
from copy import deepcopy

import numpy as np
import pandas as pd
from IPython.display import Image


# settaggi utilizzati per la visualizzazione di
# tabelle Pandas DataFrame
pd.set_option("display.max_column", None)
pd.set_option("display.max_rows", None)
pd.set_option("display.precision", 1)


# funzione che crea le seguenti strutture dati:
#   - creazione dizionario `cooking_chef_states_tuples` che contiene l'associazione: stato -> tupla
#   - creazione dizionario `cooking_chef_tuples_states` che contiene l'associazione: tupla -> stato
def create_problem_grid():
    cooking_chef_states_tuples = {}
    cooking_chef_tuples_states = {}
    count = 0
    for i in range(1, 9):
        for j in range(1, 10):
            if not (
                (i, j) == (1, 5)
                or (i, j) == (3, 5)
                or (i, j) == (4, 5)
                or (i, j) == (5, 5)
                or (i, j) == (7, 5)
                or (i, j) == (8, 5)
            ):
                cooking_chef_states_tuples[count] = (i, j)
                cooking_chef_tuples_states[(i, j)] = count
                count += 1
    return cooking_chef_states_tuples, cooking_chef_tuples_states


cooking_chef_states_tuples, cooking_chef_tuples_states = create_problem_grid()


# funzione che stampa a video la griglia di stati del problema.
#
# `dict_view`: parametro utilizzato per specificare se stampare
# la griglia di stati oppure la griglia di tuple che corrispondono
# alle caselle utilizzabili del problema
def print_states(dict_view=False):
    tiles = [i for i in range(0, len(cooking_chef_states_tuples))]
    row_8 = tiles[58:66]
    row_7 = tiles[50:58]
    row_6 = tiles[41:50]
    row_5 = tiles[33:41]
    row_4 = tiles[25:33]
    row_3 = tiles[17:25]
    row_2 = tiles[8:17]
    row_1 = tiles[0:8]

    rows = row_8, row_7, row_6, row_5, row_4, row_3, row_2, row_1

    if dict_view:
        for row in rows:
            for i in row:
                print(f" {cooking_chef_states_tuples[i]} ", end="")
                if (
                    cooking_chef_states_tuples[i] == (8, 4)
                    or cooking_chef_states_tuples[i] == (7, 4)
                    or cooking_chef_states_tuples[i] == (5, 4)
                    or cooking_chef_states_tuples[i] == (4, 4)
                    or cooking_chef_states_tuples[i] == (3, 4)
                    or cooking_chef_states_tuples[i] == (1, 4)
                ):
                    print("        ", end="")
            if row == row_5:
                print("\n\n")
            else:
                print()
    else:
        for row in rows:
            for i in row:
                print(f" {i:<2} ", end="")
                if (
                    cooking_chef_states_tuples[i] == (8, 4)
                    or cooking_chef_states_tuples[i] == (7, 4)
                    or cooking_chef_states_tuples[i] == (5, 4)
                    or cooking_chef_states_tuples[i] == (4, 4)
                    or cooking_chef_states_tuples[i] == (3, 4)
                    or cooking_chef_states_tuples[i] == (1, 4)
                ):
                    print("    ", end="")
            if row == row_5:
                print("\n\n")
            else:
                print()


# funzione che stampa a video la policy passata come
# parametro.
#
# `policy`: array che corrisponde alla policy da stampare
def print_policy(policy: np.ndarray = None):
    # 0 -> left : ←
    # 1 -> right: →
    # 2 -> up   : ↑
    # 3 -> down : ↓
    # 4 -> take_egg_beater : T
    # 5 -> cook_eggs : C
    actions = ["←", "→", "↑", "↓", "T", "C"]

    tiles = [i for i in range(0, len(cooking_chef_states_tuples))]
    row_8 = tiles[58:66]
    row_7 = tiles[50:58]
    row_6 = tiles[41:50]
    row_5 = tiles[33:41]
    row_4 = tiles[25:33]
    row_3 = tiles[17:25]
    row_2 = tiles[8:17]
    row_1 = tiles[0:8]

    rows = row_8, row_7, row_6, row_5, row_4, row_3, row_2, row_1

    for row in rows:
        for i in row:
            print(f" {actions[policy[i]]} ", end="")
            if (
                cooking_chef_states_tuples[i] == (8, 4)
                or cooking_chef_states_tuples[i] == (7, 4)
                or cooking_chef_states_tuples[i] == (5, 4)
                or cooking_chef_states_tuples[i] == (4, 4)
                or cooking_chef_states_tuples[i] == (3, 4)
                or cooking_chef_states_tuples[i] == (1, 4)
            ):
                print("   ", end="")
        if row == row_5:
            print("\n\n")
        else:
            print()


# funzione che stampa a video la state value function
# della policy pi passata come parametro.
#
# `V_pi`: array che corrisponde alla state value function
# della policy pi da stampare
def print_state_value_function(V_pi: np.ndarray = None):
    tiles = [i for i in range(0, len(cooking_chef_states_tuples))]
    row_8 = tiles[58:66]
    row_7 = tiles[50:58]
    row_6 = tiles[41:50]
    row_5 = tiles[33:41]
    row_4 = tiles[25:33]
    row_3 = tiles[17:25]
    row_2 = tiles[8:17]
    row_1 = tiles[0:8]

    rows = row_8, row_7, row_6, row_5, row_4, row_3, row_2, row_1

    for row in rows:
        for i in row:
            print(f" {round(V_pi[i], 2):<8} ", end="")
            if (
                cooking_chef_states_tuples[i] == (8, 4)
                or cooking_chef_states_tuples[i] == (7, 4)
                or cooking_chef_states_tuples[i] == (5, 4)
                or cooking_chef_states_tuples[i] == (4, 4)
                or cooking_chef_states_tuples[i] == (3, 4)
                or cooking_chef_states_tuples[i] == (1, 4)
            ):
                print("          ", end="")
        if row == row_5:
            print("\n\n")
        else:
            print()


# funzione che esegue la L1 norm
# (Manhattan norm) del vettore `a`
# passato come parametro.
#
# `a`: np.ndarray, vettore su cui eseguire la norma
#
# return: int, norma del vettore `a`
def L1_norm(a: np.ndarray):
    l1 = 0
    for i in a:
        l1 += abs(i)
    return l1


# funzione che esegue la Infinity norm
# (Chebyshev norm) del vettore `a`
# passato come parametro.
#
# `a`: np.ndarray, vettore su cui eseguire la norma
#
# return: int, norma del vettore `a`
def Linf_norm(a: np.ndarray):
    return max(np.abs(a))


# funzione che esegue la sottrazione componente
# per componente tra i vettori `a_1` e `a_2`
#
# a_1: np.ndarray
#
# a_2: np.ndarray
#
# return: np.ndarray, vettore risultante dalla sottrazione
def subtract(a_1: np.ndarray, a_2: np.ndarray):
    return a_1 - a_2


# funzione che ritorna: argmax_a Q(s, a).
#
# Q: state-action value function
#
# s: stato su cui calcolare l'argmax_a
#
# return: int (identificatore dell'azione), argmax_a Q(s, a)
def argmax(Q: np.ndarray, s: int):
    return np.argmax(Q[s])


# classe che associa ad ogni stato l'insieme di azioni possibili
# insieme alla probabilità di eseguire ciascuna azione possibile
# dallo stato a cui è associata
class ActionsMapping:
    def __init__(self, mapping: dict):
        self.S = self.__get_states_from_mapping(mapping)
        self.A = self.__get_actions_from_mapping(mapping)
        self.mapping = self.__convert_to_numpy_array(mapping)

    # metodo che ritorna la probabilità
    # di eseguire l'azione `a` dallo stato `s`
    def probs(self, a: int, s: int):
        return self.mapping[s, a]

    # metodo che ritorna una policy inizializzata
    # randomicamente
    def random_policy(self):
        pi = np.zeros(len(self.S), dtype=np.int64)
        for s in self.S:
            pi[s] = self.get_random_action_from_state(s)
        return pi

    # metodo che ritorna un'azione randomica
    # eseguita dallo stato `s`
    def get_random_action_from_state(self, s: int):
        return np.random.choice(self.A, p=[self.probs(a, s) for a in self.A])

    # metodo che ritorna le azioni che possono essere
    # eseguite dallo stato `s`
    def get_possible_actions_from_state(self, s: int):
        actions = []
        for a in self.A:
            if self.mapping[s, a] > 0.0:
                actions.append(a)
        return actions

    # metodo che ritorna tutte le azioni contenute
    # nel parametro `mapping`
    def __get_actions_from_mapping(self, mapping: dict):
        tot_actions = set([])
        for s, actions in mapping.items():
            for a in actions:
                tot_actions.add(a)

        return sorted(list(tot_actions))

    # metodo che ritorna tutti gli stati contenuti
    # nel parametro `mapping`
    def __get_states_from_mapping(self, mapping: dict):
        return sorted(list(mapping.keys()))

    # metodo che ritorna la conversione della struttura dati
    # dict `mapping` in numpy array
    def __convert_to_numpy_array(self, mapping: dict):
        self.__check_structure(sorted(list(mapping.keys())))
        array = np.zeros(shape=(len(self.S), len(self.A)), dtype=np.float64)
        for s, actions in mapping.items():
            prob = 1 / len(actions)
            for a in actions:
                array[s, a] = prob
        return array

    # metodo che esegue il controllo della struttura
    # definita nel parametro `lst`
    def __check_structure(self, lst: list):
        count = 0
        for i in lst:
            if i != count:
                raise RuntimeError("Struttura non valida")
            count += 1


# classe che codifica un generico MDP
class MDP:
    def __init__(
        self,
        S: np.ndarray = None,
        A: np.ndarray = None,
        P: np.ndarray = None,
        R: np.ndarray = None,
        am: ActionsMapping = None,
        gamma: float = None,
    ):
        self.S = S
        self.A = A
        self.P = P
        self.R = R
        self.am = am
        self.gamma = gamma
        # valori calcolati dalla Policy Iteration
        self.state_value_function = None
        self.optimal_policy = None

    # metodo che esegue il calcolo della policy ottima
    # mediante l'algoritmo Policy Iteration.
    # Alla terminazione dell'algoritmo, vengono settate
    # le variabili `optimal_policy` e `state_value_function`
    #
    # return: np.ndarray, policy ottima trovata
    def policy_iteration(self):
        # inizializzazione
        pi = self.am.random_policy()
        pi_k = deepcopy(pi)
        i = 0
        # policy iteration
        while i == 0 or L1_norm(subtract(pi, pi_k)) > 0:
            pi_k = deepcopy(pi)
            Q = np.zeros(shape=(len(self.S), len(self.A)), dtype=np.float64)
            # valutazione policy corrente
            V = self.__policy_evaluation(pi)
            # calcolo dei Q-values per la policy corrente
            for s in self.S:
                for a in self.A:
                    Q[s, a] = 0
                    for ns in self.S:
                        Q[s, a] += (
                            self.R[a, s, ns] + self.gamma * self.P[a, s, ns] * V[ns]
                        )
            # calcolo nuova policy
            for s in self.S:
                pi[s] = argmax(Q, s)

            i += 1

        # salvataggio di V_pi e pi
        self.state_value_function = V
        self.optimal_policy = pi
        # ritorno della policy ottima trovata
        return pi

    # metodo che esegue la Policy Evaluation della
    # policy `pi` passata come parametro
    #
    # `pi`: np.ndarray, policy da valutare
    #
    # return: state value function relativa alla policy pi
    def __policy_evaluation(self, pi: np.ndarray):
        V = np.zeros(len(self.S), dtype=np.float64)
        while True:
            V_k = deepcopy(V)
            for s in self.S:
                V[s] = 0
                for ns in self.S:
                    V[s] += (
                        self.R[pi[s], s, ns]
                        + self.gamma
                        * self.am.probs(pi[s], s)
                        * self.P[pi[s], s, ns]
                        * V_k[ns]
                    )
            if Linf_norm(subtract(V, V_k)) <= 0.000001:
                break
        return V

    # metodo che ritorna un DataFrame che contiene le transizioni
    # contenute nella matrice P.
    #
    # `all_transitions`: bool, parametro che specifica se inserire
    # tutte le transizioni (se specificato True) o soltanto le transizioni
    # che hanno probabilità > 0 (se specificato False)
    #
    # return: pd.DataFrame, struttura dati che corrisponde ad una tabella
    # delle transizioni specificate
    def print_transitions(self, all_transitions=False):
        columns = [
            "Sorgente (s)",
            "Destinazione (s')",
            "Azione (a)",
            "Probabilità (P[a, s, s'])",
        ]
        data = []
        for a in self.A:
            for s in self.S:
                for ns in self.S:
                    if all_transitions or self.P[a, s, ns] > 0.0:
                        data.append(
                            {
                                columns[0]: f"s{s}",
                                columns[1]: f"s{ns}",
                                columns[2]: f"{a}",
                                columns[3]: f"{self.P[a, s, ns]}",
                            }
                        )
        return pd.DataFrame(data).astype({columns[3]: "float64"})

    # metodo che ritorna un DataFrame che corrisponde alla matrice
    # delle transizioni P rispetto all'azione `action`.
    #
    # `action`: int, parametro che identifica l'azione da cui stampare
    # una matrice che contiene tutte le coppie S x S' per l'azione
    # specificata `action`
    #
    # return: pd.DataFrame, struttura dati che corrisponde alla matrice
    # delle transizioni P rispetto all'azione specificata `action`
    def print_transitions_matrix(self, action=None):
        df = pd.DataFrame(self.P[action])

        df["Stati"] = df.index
        df["Stati"] = "s" + df["Stati"].astype(str)
        df.set_index("Stati", inplace=True)
        df.rename(columns=lambda x: f"s{x}", inplace=True)

        return df

    # metodo che ritorna un DataFrame che contiene i rewards
    # contenuti nella matrice R.
    #
    # `all_rewards`: bool, parametro che specifica se inserire
    # tutti i rewards (se specificato True) o soltanto i rewards
    # che sono != 0 (se specificato False)
    #
    # return: pd.DataFrame, struttura dati che corrisponde alla tabella
    # dei rewards specificati
    def print_rewards(self, all_rewards=False):
        columns = [
            "Sorgente (s)",
            "Destinazione (s')",
            "Azione (a)",
            "Reward (R[a, s, s'])",
        ]
        data = []
        for a in self.A:
            for s in self.S:
                for ns in self.S:
                    if all_rewards or self.R[a, s, ns] != 0.0:
                        data.append(
                            {
                                columns[0]: f"s{s}",
                                columns[1]: f"s{ns}",
                                columns[2]: f"{a}",
                                columns[3]: f"{self.R[a, s, ns]}",
                            }
                        )
        return pd.DataFrame(data).astype({columns[3]: "float64"})

    # metodo che stampa a video la policy ottima
    # trovata con la Policy Iteration e la
    # state value function associata
    def print_stats(self):
        print("Policy ottima:")
        print(self.optimal_policy)
        print()
        print("State value function:")
        print(self.state_value_function)


# classe che si occupa della generazione
# degli episodi per lo scenario Model-free
class EpisodeGenerator:
    def __init__(self, R: np.ndarray = None):
        self.R = R

    # metodo utilizzato per generare un episodio a partire
    # dalla policy `policy` passata come parametro.
    #
    # `policy`: np.ndarray, policy da cui generare l'episodio
    #
    # return: episodio generato a partire dalla policy passata come
    # parametro
    def generate_episode(self, policy: np.ndarray):
        episode = []
        for s, a in enumerate(policy):
            episode.append({"s": s, "a": a, "r": self.R[a, s]})
        return episode


# classe che codifica un generico scenario
# Model-free
class ModelFree:
    def __init__(
        self,
        S: np.ndarray = None,
        A: np.ndarray = None,
        generator: EpisodeGenerator = None,
        gamma: float = None,
    ):
        self.S = S
        self.A = A
        # generatore di episodi
        self.generator = generator
        self.gamma = gamma
        # valore calcolato da Monte Carlo Online Control
        self.found_policy = None

    # metodo che calcola una epsilon-greedy policy utilizzando i
    # parametri Q ed epsilon.
    #
    # `Q`: state-action values definiti per ogni s in S ed a in A
    #
    # `epsilon`: float, numero utilizzato nel calcolo
    #
    # return: epsilon-greedy policy calcolata
    def __epsilon_greedy(self, Q: np.ndarray, epsilon: float):
        pi = np.zeros(len(self.S), dtype=np.int64)
        for s in self.S:
            pi[s] = np.random.choice(
                [argmax(Q, s), np.random.choice(self.A)],
                p=[1 - epsilon, epsilon],
            )
        return pi

    # metodo che esegue il calcolo di una policy ottima sull'istanza Model-free
    # mediante l'algoritmo Monte Carlo Online Control / On Policy improvement.
    # Alla terminazione dell'algoritmo, viene settata
    # la variabile `found_policy`
    #
    # `max_iterations`: int, numero di iterazioni massime che l'algoritmo
    # potrà effettuare
    #
    # `status_step`: int, numero che indica all'algoritmo di stampare uno
    # status ogni <status_step> iterazioni
    #
    # return: np.ndarray, policy trovata
    def monte_carlo_online_control(
        self, max_iterations: int = 1000, status_step: int = None
    ):
        Q = np.zeros(shape=(len(self.S), len(self.A)), dtype=np.float64)
        N = np.zeros(shape=(len(self.S), len(self.A)), dtype=np.int64)
        k = 1
        epsilon = 1 / k
        pi = self.__epsilon_greedy(Q, epsilon)

        while k <= max_iterations:
            Q_k = deepcopy(Q)
            episode = self.generator.generate_episode(pi)
            G_k = 0
            visited = []
            for t in reversed(range(0, len(episode))):
                s_t = episode[t]["s"]
                a_t = episode[t]["a"]
                r_t = episode[t]["r"]
                G_k += (self.gamma ** t) * r_t

                if (s_t, a_t) not in visited:
                    # first visit per la coppia (s, a) nell'episodio k
                    N[s_t, a_t] += 1
                    Q[s_t, a_t] = Q_k[s_t, a_t] + (1 / N[s_t, a_t]) * (
                        G_k - Q_k[s_t, a_t]
                    )
                    visited.append((s_t, a_t))
            k += 1
            epsilon = 1 / k

            # Policy Improvement
            pi = self.__epsilon_greedy(Q, epsilon)

            if status_step is not None and k % status_step == 0:
                print(f"Iterazione: {k} / {max_iterations}")

        self.found_policy = pi
        return pi

    # metodo che stampa a video la policy
    # trovata con Monte Carlo Online Control
    def print_stats(self):
        print("Policy trovata:")
        print(self.found_policy)


mdp = MDP()


print_states(dict_view=True)

 (8, 1)  (8, 2)  (8, 3)  (8, 4)          (8, 6)  (8, 7)  (8, 8)  (8, 9) 
 (7, 1)  (7, 2)  (7, 3)  (7, 4)          (7, 6)  (7, 7)  (7, 8)  (7, 9) 
 (6, 1)  (6, 2)  (6, 3)  (6, 4)  (6, 5)  (6, 6)  (6, 7)  (6, 8)  (6, 9) 
 (5, 1)  (5, 2)  (5, 3)  (5, 4)          (5, 6)  (5, 7)  (5, 8)  (5, 9) 


 (4, 1)  (4, 2)  (4, 3)  (4, 4)          (4, 6)  (4, 7)  (4, 8)  (4, 9) 
 (3, 1)  (3, 2)  (3, 3)  (3, 4)          (3, 6)  (3, 7)  (3, 8)  (3, 9) 
 (2, 1)  (2, 2)  (2, 3)  (2, 4)  (2, 5)  (2, 6)  (2, 7)  (2, 8)  (2, 9) 
 (1, 1)  (1, 2)  (1, 3)  (1, 4)          (1, 6)  (1, 7)  (1, 8)  (1, 9)


Image("img/cooking_chef_mdp.png")


print_states()

 58  59  60  61      62  63  64  65 
 50  51  52  53      54  55  56  57 
 41  42  43  44  45  46  47  48  49 
 33  34  35  36      37  38  39  40 


 25  26  27  28      29  30  31  32 
 17  18  19  20      21  22  23  24 
 8   9   10  11  12  13  14  15  16 
 0   1   2   3       4   5   6   7


mdp.S = np.array(list(cooking_chef_states_tuples.keys()), dtype=np.int64)
mdp.S

array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16,
       17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33,
       34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50,
       51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65])


len(mdp.S)

66


left = 0
right = 1
up = 2
down = 3
take_egg_beater = 4
cook_eggs = 5

actions = [left, right, up, down, take_egg_beater, cook_eggs]


mdp.A = np.array(actions, dtype=np.int64)
mdp.A

array([0, 1, 2, 3, 4, 5])


len(mdp.A)

6


# associazione:
# stato -> array di azioni possibili
actions_mapping = {
    0: [1],
    1: [0, 1],
    2: [0, 1],
    3: [0, 2],
    4: [1, 2],
    5: [0, 1, 2],
    6: [0, 1],
    7: [0, 2],
    8: [1, 2],
    9: [0, 1],
    10: [0, 1],
    11: [0, 1, 2, 3],
    12: [0, 1],
    13: [0, 1, 2, 3],
    14: [0, 2, 3],
    15: [1, 2],
    16: [0, 2, 3],
    17: [3, 4],
    18: [1, 2],
    19: [0, 1, 2],
    20: [0, 2, 3],
    21: [1, 2, 3],
    22: [0, 2, 3],
    23: [1, 3, 4],
    24: [0, 3],
    25: [1],
    26: [0, 1, 3],
    27: [0, 1, 3],
    28: [0, 3],
    29: [1, 3],
    30: [0, 1, 3],
    31: [0, 1],
    32: [0],
    33: [1],
    34: [0, 1],
    35: [0, 1],
    36: [0, 2],
    37: [1, 2],
    38: [0, 1, 2],
    39: [0, 1],
    40: [0, 2],
    41: [1, 2],
    42: [0, 1],
    43: [0, 1],
    44: [0, 1, 2, 3],
    45: [0, 1],
    46: [0, 1, 2, 3],
    47: [0, 2, 3],
    48: [1, 2],
    49: [0, 2, 3],
    50: [3],
    51: [1, 2],
    52: [0, 1, 2],
    53: [0, 2, 3],
    54: [1, 2, 3],
    55: [0, 2, 3],
    56: [1, 3],
    57: [0, 3],
    58: [1, 5],
    59: [0, 1, 3],
    60: [0, 1, 3],
    61: [0, 3],
    62: [1, 3],
    63: [0, 1, 3],
    64: [0, 1, 5],
    65: [0],
}


mdp.am = ActionsMapping(actions_mapping)


Image("img/cooking_chef_mdp_part_a_transitions.png")


# funzione che, dati come parametro uno stato s ed un'azione a,
# ritorna il prossimo stato in cui si troverà l'agente.
#
# `s`: parametro che specifica lo stato in cui l'agente esegue l'azione
#
# `a`: parametro che specifica l'azione che l'agente esegue
#
# return: prossimo stato in cui l'agente si troverà dopo l'applicazione
# dell'azione
def get_next_state(s: int, a: int):
    state_tuple = cooking_chef_states_tuples[s]
    tile = None
    if a == 0:
        # azione left
        tile = (state_tuple[0], state_tuple[1] - 1)
    elif a == 1:
        # azione right
        tile = (state_tuple[0], state_tuple[1] + 1)
    elif a == 2:
        # azione up
        tile = (state_tuple[0] + 1, state_tuple[1])
    elif a == 3:
        # azione down
        tile = (state_tuple[0] - 1, state_tuple[1])
    elif a == 4:
        # azione take_egg_beater
        if s == 17:
            # azione take_egg_beater dallo stato 17 (17 -> 50)
            tile = cooking_chef_states_tuples[50]
        elif s == 23:
            # azione take_egg_beater dallo stato 23 (23 -> 56)
            tile = cooking_chef_states_tuples[56]
        else:
            raise RuntimeError("Azione take_egg_beater non valida")

    elif a == 5:
        # azione cook_eggs
        tile = state_tuple

    else:
        raise RuntimeError("Azione non valida")

    return cooking_chef_tuples_states[tile]


# generazione delle transizioni
def generate_mdp_transitions(
    actions: np.ndarray, states: np.ndarray, mapping: ActionsMapping
):
    transitions = np.zeros(
        shape=(len(actions), len(states), len(states)), dtype=np.float64
    )

    for state in states:
        for action in mapping.get_possible_actions_from_state(state):
            if mapping.probs(action, state) > 0.0:
                # l'azione `action` è possibile dallo stato `state`
                # non ci sono effetti imprevisti, c'è solo uno stato destinazione
                # raggiungibile con probabilità 1
                transitions[action, state, get_next_state(state, action)] = 1

    return transitions


mdp.P = generate_mdp_transitions(mdp.A, mdp.S, mdp.am)


# |P| = |A| x |S| x |S|
mdp.P.shape

(6, 66, 66)


mdp_part_a_transition_left = mdp.print_transitions_matrix(action=left)
mdp_part_a_transition_left.to_csv("transitions/part_a/mdp_part_a_transition_left.csv")

# ogni entry > 0 viene evidenziata
mdp_part_a_transition_left.style.apply(
    lambda x: ["background: yellow" if v > 0 else "" for v in x], axis=1
)


mdp_part_a_transition_right = mdp.print_transitions_matrix(action=right)
mdp_part_a_transition_right.to_csv("transitions/part_a/mdp_part_a_transition_right.csv")

# ogni entry > 0 viene evidenziata
mdp_part_a_transition_right.style.apply(
    lambda x: ["background: yellow" if v > 0 else "" for v in x], axis=1
)


mdp_part_a_transition_up = mdp.print_transitions_matrix(action=up)
mdp_part_a_transition_up.to_csv("transitions/part_a/mdp_part_a_transition_up.csv")

# ogni entry > 0 viene evidenziata
mdp_part_a_transition_up.style.apply(
    lambda x: ["background: yellow" if v > 0 else "" for v in x], axis=1
)


mdp_part_a_transition_down = mdp.print_transitions_matrix(action=down)
mdp_part_a_transition_down.to_csv("transitions/part_a/mdp_part_a_transition_down.csv")

# ogni entry > 0 viene evidenziata
mdp_part_a_transition_down.style.apply(
    lambda x: ["background: yellow" if v > 0 else "" for v in x], axis=1
)


mdp_part_a_transition_take_egg_beater = mdp.print_transitions_matrix(
    action=take_egg_beater
)
mdp_part_a_transition_take_egg_beater.to_csv(
    "transitions/part_a/mdp_part_a_transition_take_egg_beater.csv"
)

# ogni entry > 0 viene evidenziata
mdp_part_a_transition_take_egg_beater.style.apply(
    lambda x: ["background: yellow" if v > 0 else "" for v in x], axis=1
)


mdp_part_a_transition_cook_eggs = mdp.print_transitions_matrix(action=cook_eggs)
mdp_part_a_transition_cook_eggs.to_csv(
    "transitions/part_a/mdp_part_a_transition_cook_eggs.csv"
)

# ogni entry > 0 viene evidenziata
mdp_part_a_transition_cook_eggs.style.apply(
    lambda x: ["background: yellow" if v > 0 else "" for v in x], axis=1
)


mdp.print_transitions()


# creazione dell'insieme R[A, S, S']
mdp.R = np.zeros(shape=(len(mdp.A), len(mdp.S), len(mdp.S)), dtype=np.float64)

# premiamo l'agente se esegue l'azione cook_eggs da uno degli stati obiettivi
# (58 o 64 in base alla ricetta utilizzata).
# In questo scenario (pudding eggs) verrà settato il reward per cook_eggs
# sullo stato obiettivo 64, postazione di cucina per la ricetta pudding eggs
mdp.R[cook_eggs, 64, 64] = 1000
mdp.R[cook_eggs, 58, 58] = 0

# premiamo l'agente se esegue l'azione take_egg_beater dagli stati obiettivi
# 17 e 23
mdp.R[take_egg_beater, 17, 50] = 500
mdp.R[take_egg_beater, 23, 56] = 500

# tutti gli altri rewards vengono lasciati a 0


mdp.print_rewards()


mdp.gamma = 0.9


mdp_pi = mdp.policy_iteration()


print_policy(policy=mdp_pi)

 →  →  →  ↓     →  →  C  ← 
 ↓  →  →  ↓     ↑  ↑  →  ↓ 
 →  →  →  →  →  ↑  ↑  →  ↓ 
 →  →  →  ↑     ↑  ↑  ←  ← 


 →  →  →  ↓     ↓  ↓  ←  ← 
 T  →  →  ↓     ↓  ↓  T  ← 
 ↑  ←  ←  ←  ←  ←  ↓  ↑  ← 
 →  →  →  ↑     →  →  →  ↑


# azzeriamo il reward per la postazione di cucina
# della ricetta pudding eggs
mdp.R[cook_eggs, 64, 64] = 0

# settiamo il reward per la postazione di cucina
# della ricetta scrambled eggs
mdp.R[cook_eggs, 58, 58] = 1000


mdp_pi = mdp.policy_iteration()


print_policy(policy=mdp_pi)

 C  ←  ←  ←     ↓  ←  ←  ← 
 ↓  ↑  ←  ←     ↓  ←  →  ↓ 
 →  →  →  ↑  ←  ←  ←  →  ↓ 
 →  →  →  ↑     ↑  ←  ←  ← 


 →  →  →  ↓     ↓  ↓  ←  ← 
 T  →  →  ↓     ↓  ↓  T  ← 
 ↑  ←  ←  ←  ←  ←  ↓  ↑  ← 
 →  →  →  ↑     →  →  →  ↑


def test_multiple_gamma(list_of_gamma: list):
    policies = []
    # test delle gamma specificate
    for g in list_of_gamma:
        print()
        print(f"Gamma = {g}")
        print()
        mdp_test = MDP(mdp.S, mdp.A, mdp.P, mdp.R, mdp.am, gamma=g)
        pi_test = mdp_test.policy_iteration()
        policies.append((pi_test, g))
        print_policy(policy=pi_test)
        print()
        print("--------------------------")

    # rilevamento differenze tra le policies
    # calcolate
    diff = False
    for p1 in policies:
        for p2 in policies:
            if not np.array_equal(p1[0], p2[0]):
                diff = True
                print(
                    f"- Le policies risultanti da gamma = {p1[1]} e gamma = {p2[1]} differiscono"
                )
    if diff:
        print()
        print("!- Sono state rilevate differenze tra le policies risultanti")
    else:
        print("!- Nessuna differenza rilevata tra le policies risultanti")


test_multiple_gamma([0.1, 0.3, 0.4, 0.5, 0.8, 0.9])

Gamma = 0.1

 C  ←  ←  ←     ←  ←  ←  ← 
 ←  ↑  ←  ↑     ↓  ←  ←  ← 
 →  →  →  ↑  ←  ←  ←  ←  ← 
 ←  →  →  ↑     ↑  ←  ←  ← 


 ←  →  →  ↓     ↓  ↓  ←  ← 
 T  →  →  ↓     ↓  ↓  T  ← 
 ↑  ←  ←  ←  ←  ←  ↓  ↑  ← 
 →  →  →  ↑     →  →  →  ↑ 

--------------------------

Gamma = 0.3

 C  ←  ←  ←     ↓  ←  ←  ← 
 ↓  ↑  ←  ←     ↓  ←  ←  ← 
 →  →  →  ↑  ←  ←  ←  ←  ↓ 
 →  →  →  ↑     ↑  ←  ←  ← 


 →  →  →  ↓     ↓  ↓  ←  ← 
 T  →  →  ↓     ↓  ↓  T  ← 
 ↑  ←  ←  ←  ←  ←  ↓  ↑  ← 
 →  →  →  ↑     →  →  →  ↑ 

--------------------------

Gamma = 0.4

 C  ←  ←  ←     ↓  ←  ←  ← 
 ↓  ↑  ←  ←     ↓  ←  →  ↓ 
 →  →  →  ↑  ←  ←  ←  →  ↓ 
 →  →  →  ↑     ↑  ←  ←  ← 


 →  →  →  ↓     ↓  ↓  ←  ← 
 T  →  →  ↓     ↓  ↓  T  ← 
 ↑  ←  ←  ←  ←  ←  ↓  ↑  ← 
 →  →  →  ↑     →  →  →  ↑ 

--------------------------

Gamma = 0.5

 C  ←  ←  ←     ↓  ←  ←  ← 
 ↓  ↑  ←  ←     ↓  ←  →  ↓ 
 →  →  →  ↑  ←  ←  ←  →  ↓ 
 →  →  →  ↑     ↑  ←  ←  ← 


 →  →  →  ↓     ↓  ↓  ←  ← 
 T  →  →  ↓     ↓  ↓  T  ← 
 ↑  ←  ←  ←  ←  ←  ↓  ↑  ← 
 →  →  →  ↑     →  →  →  ↑ 

--------------------------

Gamma = 0.8

 C  ←  ←  ←     ↓  ←  ←  ← 
 ↓  ↑  ←  ↑     ↓  ←  →  ↓ 
 →  →  →  ↑  ←  ←  ←  →  ↓ 
 →  →  →  ↑     ↑  ←  ←  ← 


 →  →  →  ↓     ↓  ↓  ←  ← 
 T  →  →  ↓     ↓  ↓  T  ← 
 ↑  ←  ←  ←  ←  ←  ↓  ↑  ← 
 →  →  →  ↑     →  →  →  ↑ 

--------------------------

Gamma = 0.9

 C  ←  ←  ←     ↓  ←  ←  ← 
 ↓  ↑  ←  ←     ↓  ←  →  ↓ 
 →  →  →  ↑  ←  ←  ←  →  ↓ 
 →  →  →  ↑     ↑  ←  ←  ← 


 →  →  →  ↓     ↓  ↓  ←  ← 
 T  →  →  ↓     ↓  ↓  T  ← 
 ↑  ←  ←  ←  ←  ←  ↓  ↑  ← 
 →  →  →  ↑     →  →  →  ↑ 

--------------------------
- Le policies risultanti da gamma = 0.1 e gamma = 0.3 differiscono
- Le policies risultanti da gamma = 0.1 e gamma = 0.4 differiscono
- Le policies risultanti da gamma = 0.1 e gamma = 0.5 differiscono
- Le policies risultanti da gamma = 0.1 e gamma = 0.8 differiscono
- Le policies risultanti da gamma = 0.1 e gamma = 0.9 differiscono
- Le policies risultanti da gamma = 0.3 e gamma = 0.1 differiscono
- Le policies risultanti da gamma = 0.3 e gamma = 0.4 differiscono
- Le policies risultanti da gamma = 0.3 e gamma = 0.5 differiscono
- Le policies risultanti da gamma = 0.3 e gamma = 0.8 differiscono
- Le policies risultanti da gamma = 0.3 e gamma = 0.9 differiscono
- Le policies risultanti da gamma = 0.4 e gamma = 0.1 differiscono
- Le policies risultanti da gamma = 0.4 e gamma = 0.3 differiscono
- Le policies risultanti da gamma = 0.4 e gamma = 0.8 differiscono
- Le policies risultanti da gamma = 0.5 e gamma = 0.1 differiscono
- Le policies risultanti da gamma = 0.5 e gamma = 0.3 differiscono
- Le policies risultanti da gamma = 0.5 e gamma = 0.8 differiscono
- Le policies risultanti da gamma = 0.8 e gamma = 0.1 differiscono
- Le policies risultanti da gamma = 0.8 e gamma = 0.3 differiscono
- Le policies risultanti da gamma = 0.8 e gamma = 0.4 differiscono
- Le policies risultanti da gamma = 0.8 e gamma = 0.5 differiscono
- Le policies risultanti da gamma = 0.8 e gamma = 0.9 differiscono
- Le policies risultanti da gamma = 0.9 e gamma = 0.1 differiscono
- Le policies risultanti da gamma = 0.9 e gamma = 0.3 differiscono
- Le policies risultanti da gamma = 0.9 e gamma = 0.8 differiscono

!- Sono state rilevate differenze tra le policies risultanti


# |A|
len(mdp.A)

6


# |S|
len(mdp.S)

66


# |A|^|S|
len(mdp.A) ** len(mdp.S)

2280250319867037997421842330085227917956272625811456


mf = ModelFree()


print_states()

 58  59  60  61      62  63  64  65 
 50  51  52  53      54  55  56  57 
 41  42  43  44  45  46  47  48  49 
 33  34  35  36      37  38  39  40 


 25  26  27  28      29  30  31  32 
 17  18  19  20      21  22  23  24 
 8   9   10  11  12  13  14  15  16 
 0   1   2   3       4   5   6   7


mf.S = mdp.S
mf.S

array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16,
       17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33,
       34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50,
       51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65])


mf.A = mdp.A
mf.A

array([0, 1, 2, 3, 4, 5])


# R[A, S]
episode_rewards = np.zeros(
    shape=(len(actions), len(mf.S)),
    dtype=np.float64,
)

# reward per cook_eggs dallo stato 58
episode_rewards[cook_eggs, 58] = 100

# reward per take_egg_beater dagli stati 17 e 23
episode_rewards[take_egg_beater, 17] = 50
episode_rewards[take_egg_beater, 23] = 50

# premiamo l'agente se va in direzione degli obiettivi
episode_rewards[right, 0] = 1
episode_rewards[right, 1] = 1
episode_rewards[right, 2] = 1
episode_rewards[up, 3] = 1

episode_rewards[right, 4] = 1
episode_rewards[right, 5] = 1
episode_rewards[right, 6] = 1
episode_rewards[up, 7] = 1

episode_rewards[up, 8] = 1
episode_rewards[left, 9] = 1
episode_rewards[left, 10] = 1
episode_rewards[left, 11] = 1

episode_rewards[left, 12] = 1

episode_rewards[left, 13] = 1
episode_rewards[down, 14] = 1
episode_rewards[up, 15] = 1
episode_rewards[left, 16] = 1

episode_rewards[right, 18] = 1
episode_rewards[right, 19] = 1
episode_rewards[down, 20] = 1

episode_rewards[down, 21] = 1
episode_rewards[down, 22] = 1
episode_rewards[left, 24] = 1

episode_rewards[right, 25] = 1
episode_rewards[right, 26] = 1
episode_rewards[right, 27] = 1
episode_rewards[down, 28] = 1

episode_rewards[down, 29] = 1
episode_rewards[down, 30] = 1
episode_rewards[left, 31] = 1
episode_rewards[left, 32] = 1

episode_rewards[right, 33] = 1
episode_rewards[right, 34] = 1
episode_rewards[right, 35] = 1
episode_rewards[up, 36] = 1

episode_rewards[up, 37] = 1
episode_rewards[left, 38] = 1
episode_rewards[left, 39] = 1
episode_rewards[left, 40] = 1

episode_rewards[right, 41] = 1
episode_rewards[right, 42] = 1
episode_rewards[right, 43] = 1
episode_rewards[up, 44] = 1

episode_rewards[left, 45] = 1

episode_rewards[left, 46] = 1
episode_rewards[left, 47] = 1
episode_rewards[right, 48] = 1
episode_rewards[down, 49] = 1

episode_rewards[down, 50] = 1
episode_rewards[up, 51] = 1
episode_rewards[left, 52] = 1
episode_rewards[left, 53] = 1

episode_rewards[down, 54] = 1
episode_rewards[left, 55] = 1
episode_rewards[right, 56] = 1
episode_rewards[down, 57] = 1

episode_rewards[left, 59] = 1
episode_rewards[left, 60] = 1
episode_rewards[left, 61] = 1

episode_rewards[down, 62] = 1
episode_rewards[left, 63] = 1
episode_rewards[left, 64] = 1
episode_rewards[left, 65] = 1


# penalizziamo l'agente se esegue azioni non valide (-1)
for s in mf.S:
    for a in mf.A:
        # settiamo il reward a -1 se l'azione `a` non è valida
        # nello stato `s`
        # (ovvero quando non è possibile eseguire l'azione
        # `a` dallo stato `s`)
        if mdp.am.probs(a, s) == 0.0:
            episode_rewards[a, s] = -1

# i rimanenti rewards sono settati a 0


generator = EpisodeGenerator(episode_rewards)

mf.generator = generator


mf.gamma = 0.1


mf_pi = mf.monte_carlo_online_control(max_iterations=10000, status_step=1000)

Iterazione: 1000 / 10000
Iterazione: 2000 / 10000
Iterazione: 3000 / 10000
Iterazione: 4000 / 10000
Iterazione: 5000 / 10000
Iterazione: 6000 / 10000
Iterazione: 7000 / 10000
Iterazione: 8000 / 10000
Iterazione: 9000 / 10000
Iterazione: 10000 / 10000


print_policy(policy=mf_pi)

 C  ←  ←  ←     ↓  ←  ←  ← 
 ↓  ↑  ←  ←     ↓  ←  →  ↓ 
 →  →  →  ↑  ←  ←  ←  →  ↓ 
 →  →  →  ↑     →  ←  ←  ↑ 


 →  →  →  ↓     ↓  ←  ←  ← 
 T  →  →  ↓     ↓  ↓  T  ← 
 ↑  ←  ←  ←  ←  ←  ↓  ↑  ← 
 →  →  ←  ↑     →  →  ←  ↑


Image("img/cooking_chef_mdp_part_b_transitions.png")


mdp_err = MDP()


mdp_err.S = mdp.S


mdp_err.A = mdp.A


mdp_err.am = ActionsMapping(actions_mapping)


# definizione della struttura dati che contiene gli stati
# vicini (array posto come valore delle entries nel dizionario)
# non raggiungibili da uno specifico stato
# (chiave delle entries nel dizionario)
walls = {
    # griglia in basso
    0: [8],
    1: [9],
    2: [10],
    3: [4],
    4: [3],
    6: [15],
    8: [0],
    9: [1, 18],
    10: [2, 19],
    14: [15],
    15: [6, 14],
    17: [18, 25],
    18: [9, 17],
    19: [10],
    20: [21],
    21: [20],
    22: [23],
    23: [22, 31],
    24: [32],
    25: [17, 33],
    26: [34],
    27: [35],
    28: [29, 36],
    29: [28, 37],
    30: [38],
    31: [23, 39],
    32: [24, 40],
    # griglia in alto
    33: [25, 41],
    34: [26, 42],
    35: [27, 43],
    36: [28, 37],
    37: [29, 36],
    38: [30],
    39: [31, 48],
    40: [32],
    41: [33],
    42: [34, 51],
    43: [35, 52],
    47: [48],
    48: [39, 47],
    50: [51, 58],
    51: [42, 50],
    52: [43],
    53: [54],
    54: [53],
    55: [56],
    56: [55, 64],
    57: [65],
    58: [50],
    61: [62],
    62: [61],
    64: [56],
    65: [57],
}


# funzione che data la tupla di partenza `start_pos`
# e la tupla di arrivo `end_pos`, ritorna un booleano
# che indica se è avvenuta una collisione con un muro.
#
# `start_pos`: posizione di partenza dell'agente
#
# `end_pos`: posizione di arrivo dell'agente
#
# return: True, se è avvenuta una collisione, False altrimenti.
def wall_collision(start_pos: tuple, end_pos: tuple):
    start_state = cooking_chef_tuples_states[start_pos]
    end_state = None

    try:
        end_state = cooking_chef_tuples_states[end_pos]
    except KeyError:
        return True

    try:
        if end_state in walls[start_state]:
            return True
        else:
            return False
    except KeyError:
        return False


# funzione che data la tupla di partenza `start_pos`
# e la tupla che corrisponde al possibile arrivo `end_pos`,
# ritorna la tupla che corrisponde alla reale posizione
# di arrivo dell'agente.
#
# `start_pos`: posizione di partenza dell'agente
#
# `end_pos`: posizione di arrivo possibile dell'agente
#
# return: posizione reale di arrivo dell'agente
def new_tile_pos(start_pos: tuple, end_pos: tuple):
    collision = wall_collision(start_pos, end_pos)

    if collision:
        # in caso di collisione, l'agente torna nella
        # posizione di partenza
        return start_pos
    else:
        # la posizione di arrivo possibile è valida, viene
        # quindi ritornata
        return end_pos


# funzione che dati come parametro uno stato `s` ed un'azione `a`
# ritorna l'insieme di prossimi stati in cui si potrebbe
# trovare l'agente considerando il fatto che l'agente è stanco
# e quindi sbaglierà strada:
# Ciascuna azione ha il:
#   - 60% di probabilità di andare perpendicolare a sinistra nella direzione
#         scelta
#   - 40% di probabilità di andare perpendicolare a destra nella direzione
#         scelta
#
# `s`: parametro che specifica lo stato in cui l'agente esegue l'azione
#
# `a`: parametro che specifica l'azione che l'agente esegue
#
# return: insieme di tuple che contengono i prossimi stati possibili
# in cui l'agente si troverà a seguito dell'applicazione dell'azione
# insieme alle relative probabilità
def get_next_state_errors(s: int, a: int):
    state_tuple = cooking_chef_states_tuples[s]
    tiles = []
    if a == 0:
        # azione sinistra
        # risultato: 60% sotto, 40% sopra
        tile_down = new_tile_pos(state_tuple, (state_tuple[0] - 1, state_tuple[1]))

        tile_up = new_tile_pos(state_tuple, (state_tuple[0] + 1, state_tuple[1]))

        tiles.append((tile_down, 0.6))
        tiles.append((tile_up, 0.4))

    elif a == 1:
        # azione destra
        # risultato: 60% sopra, 40% sotto
        tile_up = new_tile_pos(state_tuple, (state_tuple[0] + 1, state_tuple[1]))

        tile_down = new_tile_pos(state_tuple, (state_tuple[0] - 1, state_tuple[1]))

        tiles.append((tile_up, 0.6))
        tiles.append((tile_down, 0.4))

    elif a == 2:
        # azione sopra
        # risultato: 60% sinistra, 40% destra
        tile_left = new_tile_pos(state_tuple, (state_tuple[0], state_tuple[1] - 1))

        tile_right = new_tile_pos(state_tuple, (state_tuple[0], state_tuple[1] + 1))

        tiles.append((tile_left, 0.6))
        tiles.append((tile_right, 0.4))

    elif a == 3:
        # azione sotto
        # risultato: 60% destra, 40% sinistra
        tile_right = new_tile_pos(state_tuple, (state_tuple[0], state_tuple[1] + 1))

        tile_left = new_tile_pos(state_tuple, (state_tuple[0], state_tuple[1] - 1))

        tiles.append((tile_right, 0.6))
        tiles.append((tile_left, 0.4))

    elif a == 4:
        # azione take_egg_beater (stati: 17 or 23), non ci sono errori
        # per questa azione
        if s == 17:
            tiles.append((cooking_chef_states_tuples[50], 1.0))
        elif s == 23:
            tiles.append((cooking_chef_states_tuples[56], 1.0))
        else:
            raise RuntimeError("Azione take_egg_beater non valida")

    elif a == 5:
        # azione cook_eggs, non ci sono errori per questa azione
        tiles.append((state_tuple, 1.0))

    else:
        raise RuntimeError("Azione non valida")

    # lancia un'eccezione se avviene un KeyError
    return [(cooking_chef_tuples_states[tile[0]], tile[1]) for tile in tiles]


# funzione che analizza le transizioni passate come parametro
# e, se due transizioni hanno come componenti: lo stesso stato,
# azione e nuovo stato, allora le probabilità contenute nelle due
# transizioni vengono sommate.
#
# `transitions`: array che contiene le transizioni da filtrare
#
# return: array di tuple che corrisponde alle transizioni filtrate
def filter_transitions(transitions: list):
    results = {}
    for transition in transitions:
        state, action, new_state, state_prob = transition
        try:
            results[(state, action, new_state)] += state_prob
        except KeyError:
            results[(state, action, new_state)] = state_prob

    return [(key[0], key[1], key[2], state_prob) for key, state_prob in results.items()]


# P[A, S, S']


def generate_stochastic_mdp_transitions(
    actions: np.ndarray, states: np.ndarray, mapping: ActionsMapping
):
    stochastic_transitions = np.zeros(
        shape=(len(actions), len(states), len(states)), dtype=np.float64
    )

    for state in states:
        for action in mapping.get_possible_actions_from_state(state):
            if mapping.probs(action, state) > 0.0:
                # l'azione `action` è possibile dallo stato `state`
                # ma ci sono effetti imprevisti: più stati destinazione
                # raggiungibili con diverse probabilità
                valid_transitions = []
                for new_state_with_prob in get_next_state_errors(state, action):
                    new_state, state_prob = new_state_with_prob
                    # inseriamo nella lista `valid_transitions` tutte
                    # le transizioni possibili a seguito
                    # dell'applicazione dell'azione valida `action`
                    valid_transitions.append((state, action, new_state, state_prob))

                # inseriamo le transizioni filtrate in P
                for transition in filter_transitions(valid_transitions):
                    s, a, ns, sp = transition
                    stochastic_transitions[a, s, ns] = sp
    return stochastic_transitions


mdp_err.P = generate_stochastic_mdp_transitions(mdp_err.A, mdp_err.S, mdp_err.am)


mdp_err.R = mdp.R


mdp_err.gamma = mdp.gamma


mdp_err_pi = mdp_err.policy_iteration()


print_policy(policy=mdp_err_pi)

 C  ↓  ↓  ↓     ←  ←  ←  ← 
 ←  →  ↑  ↑     ←  ←  ←  ← 
 ←  ←  ←  →  ←  ←  ←  ←  ← 
 ←  ←  ←  ←     ←  ←  ←  ← 


 ←  ←  ←  ←     ←  ←  ←  ← 
 T  ←  ←  ←     ←  ←  T  ↓ 
 →  ←  ←  ←  ←  ←  ←  →  ↑ 
 ←  ←  ←  ←     ←  ←  ←  ←


mdp_part_b_transition_left = mdp_err.print_transitions_matrix(action=left)
mdp_part_b_transition_left.to_csv("transitions/part_b/mdp_part_b_transition_left.csv")

# ogni entry > 0 viene evidenziata
mdp_part_b_transition_left.style.apply(
    lambda x: ["background: yellow" if v > 0 else "" for v in x], axis=1
)


mdp_part_b_transition_right = mdp_err.print_transitions_matrix(action=right)
mdp_part_b_transition_right.to_csv("transitions/part_b/mdp_part_b_transition_right.csv")

# ogni entry > 0 viene evidenziata
mdp_part_b_transition_right.style.apply(
    lambda x: ["background: yellow" if v > 0 else "" for v in x], axis=1
)


mdp_part_b_transition_up = mdp_err.print_transitions_matrix(action=up)
mdp_part_b_transition_up.to_csv("transitions/part_b/mdp_part_b_transition_up.csv")

# ogni entry > 0 viene evidenziata
mdp_part_b_transition_up.style.apply(
    lambda x: ["background: yellow" if v > 0 else "" for v in x], axis=1
)


mdp_part_b_transition_down = mdp_err.print_transitions_matrix(action=down)
mdp_part_b_transition_down.to_csv("transitions/part_b/mdp_part_b_transition_down.csv")

# ogni entry > 0 viene evidenziata
mdp_part_b_transition_down.style.apply(
    lambda x: ["background: yellow" if v > 0 else "" for v in x], axis=1
)


mdp_part_b_transition_take_egg_beater = mdp_err.print_transitions_matrix(
    action=take_egg_beater
)
mdp_part_b_transition_take_egg_beater.to_csv(
    "transitions/part_b/mdp_part_b_transition_take_egg_beater.csv"
)

# ogni entry > 0 viene evidenziata
mdp_part_b_transition_take_egg_beater.style.apply(
    lambda x: ["background: yellow" if v > 0 else "" for v in x], axis=1
)


mdp_part_b_transition_cook_eggs = mdp_err.print_transitions_matrix(action=cook_eggs)
mdp_part_b_transition_cook_eggs.to_csv(
    "transitions/part_b/mdp_part_b_transition_cook_eggs.csv"
)

# ogni entry > 0 viene evidenziata
mdp_part_b_transition_cook_eggs.style.apply(
    lambda x: ["background: yellow" if v > 0 else "" for v in x], axis=1
)


mdp_err.print_transitions()


print_policy(policy=mdp.optimal_policy)

 C  ←  ←  ←     ↓  ←  ←  ← 
 ↓  ↑  ←  ←     ↓  ←  →  ↓ 
 →  →  →  ↑  ←  ←  ←  →  ↓ 
 →  →  →  ↑     ↑  ←  ←  ← 


 →  →  →  ↓     ↓  ↓  ←  ← 
 T  →  →  ↓     ↓  ↓  T  ← 
 ↑  ←  ←  ←  ←  ←  ↓  ↑  ← 
 →  →  →  ↑     →  →  →  ↑


print_policy(policy=mdp_err.optimal_policy)

 C  ↓  ↓  ↓     ←  ←  ←  ← 
 ←  →  ↑  ↑     ←  ←  ←  ← 
 ←  ←  ←  →  ←  ←  ←  ←  ← 
 ←  ←  ←  ←     ←  ←  ←  ← 


 ←  ←  ←  ←     ←  ←  ←  ← 
 T  ←  ←  ←     ←  ←  T  ↓ 
 →  ←  ←  ←  ←  ←  ←  →  ↑ 
 ←  ←  ←  ←     ←  ←  ←  ←


mdp.state_value_function

array([8.41064489e-01, 9.34516241e-01, 2.07670290e+00, 4.61489548e+00,
       1.84528217e+00, 4.10062705e+00, 1.36687568e+01, 3.03750152e+01,
       2.25082547e+02, 1.01287146e+02, 4.55792155e+01, 1.02553234e+01,
       4.61489548e+00, 1.03835145e+00, 1.23018811e+00, 2.25000113e+02,
       6.75000338e+01, 5.00183437e+02, 4.15340552e-01, 9.22979067e-01,
       3.07659699e+00, 3.11505414e-01, 3.69056432e-01, 5.00000251e+02,
       2.25000113e+02, 1.12141894e-01, 1.24602147e-01, 4.15340552e-01,
       1.38446860e+00, 1.40177415e-01, 1.10716928e-01, 4.98226165e-02,
       4.48403523e-02, 4.07639170e-01, 4.52932554e-01, 1.00651693e+00,
       2.23670443e+00, 2.26466277e-01, 6.79398617e-02, 3.05729164e-02,
       1.37577910e-02, 4.52932554e-01, 1.00651693e+00, 2.23670443e+00,
       4.97045443e+00, 2.23670443e+00, 5.03258464e-01, 1.50977518e-01,
       1.85728110e-03, 4.12732303e-03, 4.07639170e-01, 2.45454545e+02,
       7.36363633e+01, 2.20909089e+01, 1.50977518e-01, 4.52932411e-02,
       8.35762231e-04, 1.85728110e-03, 1.81818182e+03, 5.45454545e+02,
       1.63636363e+02, 7.36363633e+01, 6.79398617e-02, 2.03819442e-02,
       6.11457376e-03, 5.50309737e-03])


print_state_value_function(V_pi=mdp.state_value_function)

 1818.18   545.45    163.64    73.64               0.07      0.02      0.01      0.01     
 0.41      245.45    73.64     22.09               0.15      0.05      0.0       0.0      
 0.45      1.01      2.24      4.97      2.24      0.5       0.15      0.0       0.0      
 0.41      0.45      1.01      2.24                0.23      0.07      0.03      0.01     


 0.11      0.12      0.42      1.38                0.14      0.11      0.05      0.04     
 500.18    0.42      0.92      3.08                0.31      0.37      500.0     225.0    
 225.08    101.29    45.58     10.26     4.61      1.04      1.23      225.0     67.5     
 0.84      0.93      2.08      4.61                1.85      4.1       13.67     30.38


sum(mdp.state_value_function)

4931.509575386304


mdp_err.state_value_function

array([0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 8.30346566e+00,
       1.64634146e+02, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 1.64634146e+02,
       3.36751663e+01, 5.00000000e+02, 0.00000000e+00, 0.00000000e+00,
       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 5.00000000e+02,
       1.23287671e+02, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
       9.44457631e-02, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
       3.83030113e-01, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 7.35017296e+01,
       1.35632268e+01, 2.77429636e+00, 0.00000000e+00, 0.00000000e+00,
       0.00000000e+00, 0.00000000e+00, 1.81818182e+03, 2.23227476e+02,
       2.80314318e+01, 6.91185981e+00, 0.00000000e+00, 0.00000000e+00,
       0.00000000e+00, 0.00000000e+00])


print_state_value_function(V_pi=mdp_err.state_value_function)

 1818.18   223.23    28.03     6.91                0.0       0.0       0.0       0.0      
 0.0       73.5      13.56     2.77                0.0       0.0       0.0       0.0      
 0.0       0.0       0.0       0.38      0.0       0.0       0.0       0.0       0.0      
 0.0       0.0       0.0       0.09                0.0       0.0       0.0       0.0      


 0.0       0.0       0.0       0.0                 0.0       0.0       0.0       0.0      
 500.0     0.0       0.0       0.0                 0.0       0.0       500.0     123.29   
 164.63    0.0       0.0       0.0       0.0       0.0       0.0       164.63    33.68    
 0.0       0.0       0.0       0.0                 0.0       0.0       0.0       8.3


sum(mdp_err.state_value_function)

3661.203909211263

	Sorgente (s)	Destinazione (s')	Azione (a)	Probabilità (P[a, s, s'])
0	s1	s0	0	1.0
1	s2	s1	0	1.0
2	s3	s2	0	1.0
3	s5	s4	0	1.0
4	s6	s5	0	1.0
5	s7	s6	0	1.0
6	s9	s8	0	1.0
7	s10	s9	0	1.0
8	s11	s10	0	1.0
9	s12	s11	0	1.0
10	s13	s12	0	1.0
11	s14	s13	0	1.0
12	s16	s15	0	1.0
13	s19	s18	0	1.0
14	s20	s19	0	1.0
15	s22	s21	0	1.0
16	s24	s23	0	1.0
17	s26	s25	0	1.0
18	s27	s26	0	1.0
19	s28	s27	0	1.0
20	s30	s29	0	1.0
21	s31	s30	0	1.0
22	s32	s31	0	1.0
23	s34	s33	0	1.0
24	s35	s34	0	1.0
25	s36	s35	0	1.0
26	s38	s37	0	1.0
27	s39	s38	0	1.0
28	s40	s39	0	1.0
29	s42	s41	0	1.0
30	s43	s42	0	1.0
31	s44	s43	0	1.0
32	s45	s44	0	1.0
33	s46	s45	0	1.0
34	s47	s46	0	1.0
35	s49	s48	0	1.0
36	s52	s51	0	1.0
37	s53	s52	0	1.0
38	s55	s54	0	1.0
39	s57	s56	0	1.0
40	s59	s58	0	1.0
41	s60	s59	0	1.0
42	s61	s60	0	1.0
43	s63	s62	0	1.0
44	s64	s63	0	1.0
45	s65	s64	0	1.0
46	s0	s1	1	1.0
47	s1	s2	1	1.0
48	s2	s3	1	1.0
49	s4	s5	1	1.0
50	s5	s6	1	1.0
51	s6	s7	1	1.0
52	s8	s9	1	1.0
53	s9	s10	1	1.0
54	s10	s11	1	1.0
55	s11	s12	1	1.0
56	s12	s13	1	1.0
57	s13	s14	1	1.0
58	s15	s16	1	1.0
59	s18	s19	1	1.0
60	s19	s20	1	1.0
61	s21	s22	1	1.0
62	s23	s24	1	1.0
63	s25	s26	1	1.0
64	s26	s27	1	1.0
65	s27	s28	1	1.0
66	s29	s30	1	1.0
67	s30	s31	1	1.0
68	s31	s32	1	1.0
69	s33	s34	1	1.0
70	s34	s35	1	1.0
71	s35	s36	1	1.0
72	s37	s38	1	1.0
73	s38	s39	1	1.0
74	s39	s40	1	1.0
75	s41	s42	1	1.0
76	s42	s43	1	1.0
77	s43	s44	1	1.0
78	s44	s45	1	1.0
79	s45	s46	1	1.0
80	s46	s47	1	1.0
81	s48	s49	1	1.0
82	s51	s52	1	1.0
83	s52	s53	1	1.0
84	s54	s55	1	1.0
85	s56	s57	1	1.0
86	s58	s59	1	1.0
87	s59	s60	1	1.0
88	s60	s61	1	1.0
89	s62	s63	1	1.0
90	s63	s64	1	1.0
91	s64	s65	1	1.0
92	s3	s11	2	1.0
93	s4	s13	2	1.0
94	s5	s14	2	1.0
95	s7	s16	2	1.0
96	s8	s17	2	1.0
97	s11	s20	2	1.0
98	s13	s21	2	1.0
99	s14	s22	2	1.0
100	s15	s23	2	1.0
101	s16	s24	2	1.0
102	s18	s26	2	1.0
103	s19	s27	2	1.0
104	s20	s28	2	1.0
105	s21	s29	2	1.0
106	s22	s30	2	1.0
107	s36	s44	2	1.0
108	s37	s46	2	1.0
109	s38	s47	2	1.0
110	s40	s49	2	1.0
111	s41	s50	2	1.0
112	s44	s53	2	1.0
113	s46	s54	2	1.0
114	s47	s55	2	1.0
115	s48	s56	2	1.0
116	s49	s57	2	1.0
117	s51	s59	2	1.0
118	s52	s60	2	1.0
119	s53	s61	2	1.0
120	s54	s62	2	1.0
121	s55	s63	2	1.0
122	s11	s3	3	1.0
123	s13	s4	3	1.0
124	s14	s5	3	1.0
125	s16	s7	3	1.0
126	s17	s8	3	1.0
127	s20	s11	3	1.0
128	s21	s13	3	1.0
129	s22	s14	3	1.0
130	s23	s15	3	1.0
131	s24	s16	3	1.0
132	s26	s18	3	1.0
133	s27	s19	3	1.0
134	s28	s20	3	1.0
135	s29	s21	3	1.0
136	s30	s22	3	1.0
137	s44	s36	3	1.0
138	s46	s37	3	1.0
139	s47	s38	3	1.0
140	s49	s40	3	1.0
141	s50	s41	3	1.0
142	s53	s44	3	1.0
143	s54	s46	3	1.0
144	s55	s47	3	1.0
145	s56	s48	3	1.0
146	s57	s49	3	1.0
147	s59	s51	3	1.0
148	s60	s52	3	1.0
149	s61	s53	3	1.0
150	s62	s54	3	1.0
151	s63	s55	3	1.0
152	s17	s50	4	1.0
153	s23	s56	4	1.0
154	s58	s58	5	1.0
155	s64	s64	5	1.0

	Sorgente (s)	Destinazione (s')	Azione (a)	Reward (R[a, s, s'])
0	s17	s50	4	500.0
1	s23	s56	4	500.0
2	s64	s64	5	1000.0

	s0	s1	s2	s3	s4	s5	s6	s7	s8	s9	s10	s11	s12	s13	s14	s15	s16	s17	s18	s19	s20	s21	s22	s23	s24	s25	s26	s27	s28	s29	s30	s31	s32	s33	s34	s35	s36	s37	s38	s39	s40	s41	s42	s43	s44	s45	s46	s47	s48	s49	s50	s51	s52	s53	s54	s55	s56	s57	s58	s59	s60	s61	s62	s63	s64	s65
Stati
s0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s1	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s2	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s3	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s4	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s5	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s6	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s7	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s8	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s9	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s10	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s11	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s12	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s13	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s14	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s15	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s16	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s17	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s18	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s19	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s20	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s21	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s22	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s23	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s24	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s25	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s26	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s27	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s28	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s29	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s30	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s31	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s32	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s33	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s34	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s35	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s36	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s37	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s38	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s39	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s40	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s41	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s42	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s43	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s44	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s45	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s46	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s47	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s48	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s49	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s50	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s51	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s52	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s53	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s54	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s55	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s56	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s57	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s58	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s59	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s60	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	0.0
s61	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0
s62	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0
s63	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0
s64	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0
s65	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0

	Sorgente (s)	Destinazione (s')	Azione (a)	Probabilità (P[a, s, s'])
0	s1	s0	0	1.0
1	s2	s1	0	1.0
2	s3	s2	0	1.0
3	s5	s4	0	1.0
4	s6	s5	0	1.0
5	s7	s6	0	1.0
6	s9	s8	0	1.0
7	s10	s9	0	1.0
8	s11	s10	0	1.0
9	s12	s11	0	1.0
10	s13	s12	0	1.0
11	s14	s13	0	1.0
12	s16	s15	0	1.0
13	s19	s18	0	1.0
14	s20	s19	0	1.0
15	s22	s21	0	1.0
16	s24	s23	0	1.0
17	s26	s25	0	1.0
18	s27	s26	0	1.0
19	s28	s27	0	1.0
20	s30	s29	0	1.0
21	s31	s30	0	1.0
22	s32	s31	0	1.0
23	s34	s33	0	1.0
24	s35	s34	0	1.0
25	s36	s35	0	1.0
26	s38	s37	0	1.0
27	s39	s38	0	1.0
28	s40	s39	0	1.0
29	s42	s41	0	1.0
30	s43	s42	0	1.0
31	s44	s43	0	1.0
32	s45	s44	0	1.0
33	s46	s45	0	1.0
34	s47	s46	0	1.0
35	s49	s48	0	1.0
36	s52	s51	0	1.0
37	s53	s52	0	1.0
38	s55	s54	0	1.0
39	s57	s56	0	1.0
40	s59	s58	0	1.0
41	s60	s59	0	1.0
42	s61	s60	0	1.0
43	s63	s62	0	1.0
44	s64	s63	0	1.0
45	s65	s64	0	1.0
46	s0	s1	1	1.0
47	s1	s2	1	1.0
48	s2	s3	1	1.0
49	s4	s5	1	1.0
50	s5	s6	1	1.0
51	s6	s7	1	1.0
52	s8	s9	1	1.0
53	s9	s10	1	1.0
54	s10	s11	1	1.0
55	s11	s12	1	1.0
56	s12	s13	1	1.0
57	s13	s14	1	1.0
58	s15	s16	1	1.0
59	s18	s19	1	1.0
60	s19	s20	1	1.0
61	s21	s22	1	1.0
62	s23	s24	1	1.0
63	s25	s26	1	1.0
64	s26	s27	1	1.0
65	s27	s28	1	1.0
66	s29	s30	1	1.0
67	s30	s31	1	1.0
68	s31	s32	1	1.0
69	s33	s34	1	1.0
70	s34	s35	1	1.0
71	s35	s36	1	1.0
72	s37	s38	1	1.0
73	s38	s39	1	1.0
74	s39	s40	1	1.0
75	s41	s42	1	1.0
76	s42	s43	1	1.0
77	s43	s44	1	1.0
78	s44	s45	1	1.0
79	s45	s46	1	1.0
80	s46	s47	1	1.0
81	s48	s49	1	1.0
82	s51	s52	1	1.0
83	s52	s53	1	1.0
84	s54	s55	1	1.0
85	s56	s57	1	1.0
86	s58	s59	1	1.0
87	s59	s60	1	1.0
88	s60	s61	1	1.0
89	s62	s63	1	1.0
90	s63	s64	1	1.0
91	s64	s65	1	1.0
92	s3	s11	2	1.0
93	s4	s13	2	1.0
94	s5	s14	2	1.0
95	s7	s16	2	1.0
96	s8	s17	2	1.0
97	s11	s20	2	1.0
98	s13	s21	2	1.0
99	s14	s22	2	1.0
100	s15	s23	2	1.0
101	s16	s24	2	1.0
102	s18	s26	2	1.0
103	s19	s27	2	1.0
104	s20	s28	2	1.0
105	s21	s29	2	1.0
106	s22	s30	2	1.0
107	s36	s44	2	1.0
108	s37	s46	2	1.0
109	s38	s47	2	1.0
110	s40	s49	2	1.0
111	s41	s50	2	1.0
112	s44	s53	2	1.0
113	s46	s54	2	1.0
114	s47	s55	2	1.0
115	s48	s56	2	1.0
116	s49	s57	2	1.0
117	s51	s59	2	1.0
118	s52	s60	2	1.0
119	s53	s61	2	1.0
120	s54	s62	2	1.0
121	s55	s63	2	1.0
122	s11	s3	3	1.0
123	s13	s4	3	1.0
124	s14	s5	3	1.0
125	s16	s7	3	1.0
126	s17	s8	3	1.0
127	s20	s11	3	1.0
128	s21	s13	3	1.0
129	s22	s14	3	1.0
130	s23	s15	3	1.0
131	s24	s16	3	1.0
132	s26	s18	3	1.0
133	s27	s19	3	1.0
134	s28	s20	3	1.0
135	s29	s21	3	1.0
136	s30	s22	3	1.0
137	s44	s36	3	1.0
138	s46	s37	3	1.0
139	s47	s38	3	1.0
140	s49	s40	3	1.0
141	s50	s41	3	1.0
142	s53	s44	3	1.0
143	s54	s46	3	1.0
144	s55	s47	3	1.0
145	s56	s48	3	1.0
146	s57	s49	3	1.0
147	s59	s51	3	1.0
148	s60	s52	3	1.0
149	s61	s53	3	1.0
150	s62	s54	3	1.0
151	s63	s55	3	1.0
152	s17	s50	4	1.0
153	s23	s56	4	1.0
154	s58	s58	5	1.0
155	s64	s64	5	1.0

	Sorgente (s)	Destinazione (s')	Azione (a)	Probabilità (P[a, s, s'])
0	s1	s1	0	1.0
1	s2	s2	0	1.0
2	s3	s3	0	0.6
3	s3	s11	0	0.4
4	s5	s5	0	0.6
5	s5	s14	0	0.4
6	s6	s6	0	1.0
7	s7	s7	0	0.6
8	s7	s16	0	0.4
9	s9	s9	0	1.0
10	s10	s10	0	1.0
11	s11	s3	0	0.6
12	s11	s20	0	0.4
13	s12	s12	0	1.0
14	s13	s4	0	0.6
15	s13	s21	0	0.4
16	s14	s5	0	0.6
17	s14	s22	0	0.4
18	s16	s7	0	0.6
19	s16	s24	0	0.4
20	s19	s19	0	0.6
21	s19	s27	0	0.4
22	s20	s11	0	0.6
23	s20	s28	0	0.4
24	s22	s14	0	0.6
25	s22	s30	0	0.4
26	s24	s16	0	0.6
27	s24	s24	0	0.4
28	s26	s18	0	0.6
29	s26	s26	0	0.4
30	s27	s19	0	0.6
31	s27	s27	0	0.4
32	s28	s20	0	0.6
33	s28	s28	0	0.4
34	s30	s22	0	0.6
35	s30	s30	0	0.4
36	s31	s31	0	1.0
37	s32	s32	0	1.0
38	s34	s34	0	1.0
39	s35	s35	0	1.0
40	s36	s36	0	0.6
41	s36	s44	0	0.4
42	s38	s38	0	0.6
43	s38	s47	0	0.4
44	s39	s39	0	1.0
45	s40	s40	0	0.6
46	s40	s49	0	0.4
47	s42	s42	0	1.0
48	s43	s43	0	1.0
49	s44	s36	0	0.6
50	s44	s53	0	0.4
51	s45	s45	0	1.0
52	s46	s37	0	0.6
53	s46	s54	0	0.4
54	s47	s38	0	0.6
55	s47	s55	0	0.4
56	s49	s40	0	0.6
57	s49	s57	0	0.4
58	s52	s52	0	0.6
59	s52	s60	0	0.4
60	s53	s44	0	0.6
61	s53	s61	0	0.4
62	s55	s47	0	0.6
63	s55	s63	0	0.4
64	s57	s49	0	0.6
65	s57	s57	0	0.4
66	s59	s51	0	0.6
67	s59	s59	0	0.4
68	s60	s52	0	0.6
69	s60	s60	0	0.4
70	s61	s53	0	0.6
71	s61	s61	0	0.4
72	s63	s55	0	0.6
73	s63	s63	0	0.4
74	s64	s64	0	1.0
75	s65	s65	0	1.0
76	s0	s0	1	1.0
77	s1	s1	1	1.0
78	s2	s2	1	1.0
79	s4	s4	1	0.4
80	s4	s13	1	0.6
81	s5	s5	1	0.4
82	s5	s14	1	0.6
83	s6	s6	1	1.0
84	s8	s8	1	0.4
85	s8	s17	1	0.6
86	s9	s9	1	1.0
87	s10	s10	1	1.0
88	s11	s3	1	0.4
89	s11	s20	1	0.6
90	s12	s12	1	1.0
91	s13	s4	1	0.4
92	s13	s21	1	0.6
93	s15	s15	1	0.4
94	s15	s23	1	0.6
95	s18	s18	1	0.4
96	s18	s26	1	0.6
97	s19	s19	1	0.4
98	s19	s27	1	0.6
99	s21	s13	1	0.4
100	s21	s29	1	0.6
101	s23	s15	1	0.4
102	s23	s23	1	0.6
103	s25	s25	1	1.0
104	s26	s18	1	0.4
105	s26	s26	1	0.6
106	s27	s19	1	0.4
107	s27	s27	1	0.6
108	s29	s21	1	0.4
109	s29	s29	1	0.6
110	s30	s22	1	0.4
111	s30	s30	1	0.6
112	s31	s31	1	1.0
113	s33	s33	1	1.0
114	s34	s34	1	1.0
115	s35	s35	1	1.0
116	s37	s37	1	0.4
117	s37	s46	1	0.6
118	s38	s38	1	0.4
119	s38	s47	1	0.6
120	s39	s39	1	1.0
121	s41	s41	1	0.4
122	s41	s50	1	0.6
123	s42	s42	1	1.0
124	s43	s43	1	1.0
125	s44	s36	1	0.4
126	s44	s53	1	0.6
127	s45	s45	1	1.0
128	s46	s37	1	0.4
129	s46	s54	1	0.6
130	s48	s48	1	0.4
131	s48	s56	1	0.6
132	s51	s51	1	0.4
133	s51	s59	1	0.6
134	s52	s52	1	0.4
135	s52	s60	1	0.6
136	s54	s46	1	0.4
137	s54	s62	1	0.6
138	s56	s48	1	0.4
139	s56	s56	1	0.6
140	s58	s58	1	1.0
141	s59	s51	1	0.4
142	s59	s59	1	0.6
143	s60	s52	1	0.4
144	s60	s60	1	0.6
145	s62	s54	1	0.4
146	s62	s62	1	0.6
147	s63	s55	1	0.4
148	s63	s63	1	0.6
149	s64	s64	1	1.0
150	s3	s2	2	0.6
151	s3	s3	2	0.4
152	s4	s4	2	0.6
153	s4	s5	2	0.4
154	s5	s4	2	0.6
155	s5	s6	2	0.4
156	s7	s6	2	0.6
157	s7	s7	2	0.4
158	s8	s8	2	0.6
159	s8	s9	2	0.4
160	s11	s10	2	0.6
161	s11	s12	2	0.4
162	s13	s12	2	0.6
163	s13	s14	2	0.4
164	s14	s13	2	0.6
165	s14	s14	2	0.4
166	s15	s15	2	0.6
167	s15	s16	2	0.4
168	s16	s15	2	0.6
169	s16	s16	2	0.4
170	s18	s18	2	0.6
171	s18	s19	2	0.4
172	s19	s18	2	0.6
173	s19	s20	2	0.4
174	s20	s19	2	0.6
175	s20	s20	2	0.4
176	s21	s21	2	0.6
177	s21	s22	2	0.4
178	s22	s21	2	0.6
179	s22	s22	2	0.4
180	s36	s35	2	0.6
181	s36	s36	2	0.4
182	s37	s37	2	0.6
183	s37	s38	2	0.4
184	s38	s37	2	0.6
185	s38	s39	2	0.4
186	s40	s39	2	0.6
187	s40	s40	2	0.4
188	s41	s41	2	0.6
189	s41	s42	2	0.4
190	s44	s43	2	0.6
191	s44	s45	2	0.4
192	s46	s45	2	0.6
193	s46	s47	2	0.4
194	s47	s46	2	0.6
195	s47	s47	2	0.4
196	s48	s48	2	0.6
197	s48	s49	2	0.4
198	s49	s48	2	0.6
199	s49	s49	2	0.4
200	s51	s51	2	0.6
201	s51	s52	2	0.4
202	s52	s51	2	0.6
203	s52	s53	2	0.4
204	s53	s52	2	0.6
205	s53	s53	2	0.4
206	s54	s54	2	0.6
207	s54	s55	2	0.4
208	s55	s54	2	0.6
209	s55	s55	2	0.4
210	s11	s10	3	0.4
211	s11	s12	3	0.6
212	s13	s12	3	0.4
213	s13	s14	3	0.6
214	s14	s13	3	0.4
215	s14	s14	3	0.6
216	s16	s15	3	0.4
217	s16	s16	3	0.6
218	s17	s17	3	1.0
219	s20	s19	3	0.4
220	s20	s20	3	0.6
221	s21	s21	3	0.4
222	s21	s22	3	0.6
223	s22	s21	3	0.4
224	s22	s22	3	0.6
225	s23	s23	3	0.4
226	s23	s24	3	0.6
227	s24	s23	3	0.4
228	s24	s24	3	0.6
229	s26	s25	3	0.4
230	s26	s27	3	0.6
231	s27	s26	3	0.4
232	s27	s28	3	0.6
233	s28	s27	3	0.4
234	s28	s28	3	0.6
235	s29	s29	3	0.4
236	s29	s30	3	0.6
237	s30	s29	3	0.4
238	s30	s31	3	0.6
239	s44	s43	3	0.4
240	s44	s45	3	0.6
241	s46	s45	3	0.4
242	s46	s47	3	0.6
243	s47	s46	3	0.4
244	s47	s47	3	0.6
245	s49	s48	3	0.4
246	s49	s49	3	0.6
247	s50	s50	3	1.0
248	s53	s52	3	0.4
249	s53	s53	3	0.6
250	s54	s54	3	0.4
251	s54	s55	3	0.6
252	s55	s54	3	0.4
253	s55	s55	3	0.6
254	s56	s56	3	0.4
255	s56	s57	3	0.6
256	s57	s56	3	0.4
257	s57	s57	3	0.6
258	s59	s58	3	0.4
259	s59	s60	3	0.6
260	s60	s59	3	0.4
261	s60	s61	3	0.6
262	s61	s60	3	0.4
263	s61	s61	3	0.6
264	s62	s62	3	0.4
265	s62	s63	3	0.6
266	s63	s62	3	0.4
267	s63	s64	3	0.6
268	s17	s50	4	1.0
269	s23	s56	4	1.0
270	s58	s58	5	1.0
271	s64	s64	5	1.0

	Sorgente (s)	Destinazione (s')	Azione (a)	Probabilità (P[a, s, s'])
0	s1	s0	0	1.0
1	s2	s1	0	1.0
2	s3	s2	0	1.0
3	s5	s4	0	1.0
4	s6	s5	0	1.0
5	s7	s6	0	1.0
6	s9	s8	0	1.0
7	s10	s9	0	1.0
8	s11	s10	0	1.0
9	s12	s11	0	1.0
10	s13	s12	0	1.0
11	s14	s13	0	1.0
12	s16	s15	0	1.0
13	s19	s18	0	1.0
14	s20	s19	0	1.0
15	s22	s21	0	1.0
16	s24	s23	0	1.0
17	s26	s25	0	1.0
18	s27	s26	0	1.0
19	s28	s27	0	1.0
20	s30	s29	0	1.0
21	s31	s30	0	1.0
22	s32	s31	0	1.0
23	s34	s33	0	1.0
24	s35	s34	0	1.0
25	s36	s35	0	1.0
26	s38	s37	0	1.0
27	s39	s38	0	1.0
28	s40	s39	0	1.0
29	s42	s41	0	1.0
30	s43	s42	0	1.0
31	s44	s43	0	1.0
32	s45	s44	0	1.0
33	s46	s45	0	1.0
34	s47	s46	0	1.0
35	s49	s48	0	1.0
36	s52	s51	0	1.0
37	s53	s52	0	1.0
38	s55	s54	0	1.0
39	s57	s56	0	1.0
40	s59	s58	0	1.0
41	s60	s59	0	1.0
42	s61	s60	0	1.0
43	s63	s62	0	1.0
44	s64	s63	0	1.0
45	s65	s64	0	1.0
46	s0	s1	1	1.0
47	s1	s2	1	1.0
48	s2	s3	1	1.0
49	s4	s5	1	1.0
50	s5	s6	1	1.0
51	s6	s7	1	1.0
52	s8	s9	1	1.0
53	s9	s10	1	1.0
54	s10	s11	1	1.0
55	s11	s12	1	1.0
56	s12	s13	1	1.0
57	s13	s14	1	1.0
58	s15	s16	1	1.0
59	s18	s19	1	1.0
60	s19	s20	1	1.0
61	s21	s22	1	1.0
62	s23	s24	1	1.0
63	s25	s26	1	1.0
64	s26	s27	1	1.0
65	s27	s28	1	1.0
66	s29	s30	1	1.0
67	s30	s31	1	1.0
68	s31	s32	1	1.0
69	s33	s34	1	1.0
70	s34	s35	1	1.0
71	s35	s36	1	1.0
72	s37	s38	1	1.0
73	s38	s39	1	1.0
74	s39	s40	1	1.0
75	s41	s42	1	1.0
76	s42	s43	1	1.0
77	s43	s44	1	1.0
78	s44	s45	1	1.0
79	s45	s46	1	1.0
80	s46	s47	1	1.0
81	s48	s49	1	1.0
82	s51	s52	1	1.0
83	s52	s53	1	1.0
84	s54	s55	1	1.0
85	s56	s57	1	1.0
86	s58	s59	1	1.0
87	s59	s60	1	1.0
88	s60	s61	1	1.0
89	s62	s63	1	1.0
90	s63	s64	1	1.0
91	s64	s65	1	1.0
92	s3	s11	2	1.0
93	s4	s13	2	1.0
94	s5	s14	2	1.0
95	s7	s16	2	1.0
96	s8	s17	2	1.0
97	s11	s20	2	1.0
98	s13	s21	2	1.0
99	s14	s22	2	1.0
100	s15	s23	2	1.0
101	s16	s24	2	1.0
102	s18	s26	2	1.0
103	s19	s27	2	1.0
104	s20	s28	2	1.0
105	s21	s29	2	1.0
106	s22	s30	2	1.0
107	s36	s44	2	1.0
108	s37	s46	2	1.0
109	s38	s47	2	1.0
110	s40	s49	2	1.0
111	s41	s50	2	1.0
112	s44	s53	2	1.0
113	s46	s54	2	1.0
114	s47	s55	2	1.0
115	s48	s56	2	1.0
116	s49	s57	2	1.0
117	s51	s59	2	1.0
118	s52	s60	2	1.0
119	s53	s61	2	1.0
120	s54	s62	2	1.0
121	s55	s63	2	1.0
122	s11	s3	3	1.0
123	s13	s4	3	1.0
124	s14	s5	3	1.0
125	s16	s7	3	1.0
126	s17	s8	3	1.0
127	s20	s11	3	1.0
128	s21	s13	3	1.0
129	s22	s14	3	1.0
130	s23	s15	3	1.0
131	s24	s16	3	1.0
132	s26	s18	3	1.0
133	s27	s19	3	1.0
134	s28	s20	3	1.0
135	s29	s21	3	1.0
136	s30	s22	3	1.0
137	s44	s36	3	1.0
138	s46	s37	3	1.0
139	s47	s38	3	1.0
140	s49	s40	3	1.0
141	s50	s41	3	1.0
142	s53	s44	3	1.0
143	s54	s46	3	1.0
144	s55	s47	3	1.0
145	s56	s48	3	1.0
146	s57	s49	3	1.0
147	s59	s51	3	1.0
148	s60	s52	3	1.0
149	s61	s53	3	1.0
150	s62	s54	3	1.0
151	s63	s55	3	1.0
152	s17	s50	4	1.0
153	s23	s56	4	1.0
154	s58	s58	5	1.0
155	s64	s64	5	1.0

	Sorgente (s)	Destinazione (s')	Azione (a)	Probabilità (P[a, s, s'])
0	s1	s1	0	1.0
1	s2	s2	0	1.0
2	s3	s3	0	0.6
3	s3	s11	0	0.4
4	s5	s5	0	0.6
5	s5	s14	0	0.4
6	s6	s6	0	1.0
7	s7	s7	0	0.6
8	s7	s16	0	0.4
9	s9	s9	0	1.0
10	s10	s10	0	1.0
11	s11	s3	0	0.6
12	s11	s20	0	0.4
13	s12	s12	0	1.0
14	s13	s4	0	0.6
15	s13	s21	0	0.4
16	s14	s5	0	0.6
17	s14	s22	0	0.4
18	s16	s7	0	0.6
19	s16	s24	0	0.4
20	s19	s19	0	0.6
21	s19	s27	0	0.4
22	s20	s11	0	0.6
23	s20	s28	0	0.4
24	s22	s14	0	0.6
25	s22	s30	0	0.4
26	s24	s16	0	0.6
27	s24	s24	0	0.4
28	s26	s18	0	0.6
29	s26	s26	0	0.4
30	s27	s19	0	0.6
31	s27	s27	0	0.4
32	s28	s20	0	0.6
33	s28	s28	0	0.4
34	s30	s22	0	0.6
35	s30	s30	0	0.4
36	s31	s31	0	1.0
37	s32	s32	0	1.0
38	s34	s34	0	1.0
39	s35	s35	0	1.0
40	s36	s36	0	0.6
41	s36	s44	0	0.4
42	s38	s38	0	0.6
43	s38	s47	0	0.4
44	s39	s39	0	1.0
45	s40	s40	0	0.6
46	s40	s49	0	0.4
47	s42	s42	0	1.0
48	s43	s43	0	1.0
49	s44	s36	0	0.6
50	s44	s53	0	0.4
51	s45	s45	0	1.0
52	s46	s37	0	0.6
53	s46	s54	0	0.4
54	s47	s38	0	0.6
55	s47	s55	0	0.4
56	s49	s40	0	0.6
57	s49	s57	0	0.4
58	s52	s52	0	0.6
59	s52	s60	0	0.4
60	s53	s44	0	0.6
61	s53	s61	0	0.4
62	s55	s47	0	0.6
63	s55	s63	0	0.4
64	s57	s49	0	0.6
65	s57	s57	0	0.4
66	s59	s51	0	0.6
67	s59	s59	0	0.4
68	s60	s52	0	0.6
69	s60	s60	0	0.4
70	s61	s53	0	0.6
71	s61	s61	0	0.4
72	s63	s55	0	0.6
73	s63	s63	0	0.4
74	s64	s64	0	1.0
75	s65	s65	0	1.0
76	s0	s0	1	1.0
77	s1	s1	1	1.0
78	s2	s2	1	1.0
79	s4	s4	1	0.4
80	s4	s13	1	0.6
81	s5	s5	1	0.4
82	s5	s14	1	0.6
83	s6	s6	1	1.0
84	s8	s8	1	0.4
85	s8	s17	1	0.6
86	s9	s9	1	1.0
87	s10	s10	1	1.0
88	s11	s3	1	0.4
89	s11	s20	1	0.6
90	s12	s12	1	1.0
91	s13	s4	1	0.4
92	s13	s21	1	0.6
93	s15	s15	1	0.4
94	s15	s23	1	0.6
95	s18	s18	1	0.4
96	s18	s26	1	0.6
97	s19	s19	1	0.4
98	s19	s27	1	0.6
99	s21	s13	1	0.4
100	s21	s29	1	0.6
101	s23	s15	1	0.4
102	s23	s23	1	0.6
103	s25	s25	1	1.0
104	s26	s18	1	0.4
105	s26	s26	1	0.6
106	s27	s19	1	0.4
107	s27	s27	1	0.6
108	s29	s21	1	0.4
109	s29	s29	1	0.6
110	s30	s22	1	0.4
111	s30	s30	1	0.6
112	s31	s31	1	1.0
113	s33	s33	1	1.0
114	s34	s34	1	1.0
115	s35	s35	1	1.0
116	s37	s37	1	0.4
117	s37	s46	1	0.6
118	s38	s38	1	0.4
119	s38	s47	1	0.6
120	s39	s39	1	1.0
121	s41	s41	1	0.4
122	s41	s50	1	0.6
123	s42	s42	1	1.0
124	s43	s43	1	1.0
125	s44	s36	1	0.4
126	s44	s53	1	0.6
127	s45	s45	1	1.0
128	s46	s37	1	0.4
129	s46	s54	1	0.6
130	s48	s48	1	0.4
131	s48	s56	1	0.6
132	s51	s51	1	0.4
133	s51	s59	1	0.6
134	s52	s52	1	0.4
135	s52	s60	1	0.6
136	s54	s46	1	0.4
137	s54	s62	1	0.6
138	s56	s48	1	0.4
139	s56	s56	1	0.6
140	s58	s58	1	1.0
141	s59	s51	1	0.4
142	s59	s59	1	0.6
143	s60	s52	1	0.4
144	s60	s60	1	0.6
145	s62	s54	1	0.4
146	s62	s62	1	0.6
147	s63	s55	1	0.4
148	s63	s63	1	0.6
149	s64	s64	1	1.0
150	s3	s2	2	0.6
151	s3	s3	2	0.4
152	s4	s4	2	0.6
153	s4	s5	2	0.4
154	s5	s4	2	0.6
155	s5	s6	2	0.4
156	s7	s6	2	0.6
157	s7	s7	2	0.4
158	s8	s8	2	0.6
159	s8	s9	2	0.4
160	s11	s10	2	0.6
161	s11	s12	2	0.4
162	s13	s12	2	0.6
163	s13	s14	2	0.4
164	s14	s13	2	0.6
165	s14	s14	2	0.4
166	s15	s15	2	0.6
167	s15	s16	2	0.4
168	s16	s15	2	0.6
169	s16	s16	2	0.4
170	s18	s18	2	0.6
171	s18	s19	2	0.4
172	s19	s18	2	0.6
173	s19	s20	2	0.4
174	s20	s19	2	0.6
175	s20	s20	2	0.4
176	s21	s21	2	0.6
177	s21	s22	2	0.4
178	s22	s21	2	0.6
179	s22	s22	2	0.4
180	s36	s35	2	0.6
181	s36	s36	2	0.4
182	s37	s37	2	0.6
183	s37	s38	2	0.4
184	s38	s37	2	0.6
185	s38	s39	2	0.4
186	s40	s39	2	0.6
187	s40	s40	2	0.4
188	s41	s41	2	0.6
189	s41	s42	2	0.4
190	s44	s43	2	0.6
191	s44	s45	2	0.4
192	s46	s45	2	0.6
193	s46	s47	2	0.4
194	s47	s46	2	0.6
195	s47	s47	2	0.4
196	s48	s48	2	0.6
197	s48	s49	2	0.4
198	s49	s48	2	0.6
199	s49	s49	2	0.4
200	s51	s51	2	0.6
201	s51	s52	2	0.4
202	s52	s51	2	0.6
203	s52	s53	2	0.4
204	s53	s52	2	0.6
205	s53	s53	2	0.4
206	s54	s54	2	0.6
207	s54	s55	2	0.4
208	s55	s54	2	0.6
209	s55	s55	2	0.4
210	s11	s10	3	0.4
211	s11	s12	3	0.6
212	s13	s12	3	0.4
213	s13	s14	3	0.6
214	s14	s13	3	0.4
215	s14	s14	3	0.6
216	s16	s15	3	0.4
217	s16	s16	3	0.6
218	s17	s17	3	1.0
219	s20	s19	3	0.4
220	s20	s20	3	0.6
221	s21	s21	3	0.4
222	s21	s22	3	0.6
223	s22	s21	3	0.4
224	s22	s22	3	0.6
225	s23	s23	3	0.4
226	s23	s24	3	0.6
227	s24	s23	3	0.4
228	s24	s24	3	0.6
229	s26	s25	3	0.4
230	s26	s27	3	0.6
231	s27	s26	3	0.4
232	s27	s28	3	0.6
233	s28	s27	3	0.4
234	s28	s28	3	0.6
235	s29	s29	3	0.4
236	s29	s30	3	0.6
237	s30	s29	3	0.4
238	s30	s31	3	0.6
239	s44	s43	3	0.4
240	s44	s45	3	0.6
241	s46	s45	3	0.4
242	s46	s47	3	0.6
243	s47	s46	3	0.4
244	s47	s47	3	0.6
245	s49	s48	3	0.4
246	s49	s49	3	0.6
247	s50	s50	3	1.0
248	s53	s52	3	0.4
249	s53	s53	3	0.6
250	s54	s54	3	0.4
251	s54	s55	3	0.6
252	s55	s54	3	0.4
253	s55	s55	3	0.6
254	s56	s56	3	0.4
255	s56	s57	3	0.6
256	s57	s56	3	0.4
257	s57	s57	3	0.6
258	s59	s58	3	0.4
259	s59	s60	3	0.6
260	s60	s59	3	0.4
261	s60	s61	3	0.6
262	s61	s60	3	0.4
263	s61	s61	3	0.6
264	s62	s62	3	0.4
265	s62	s63	3	0.6
266	s63	s62	3	0.4
267	s63	s64	3	0.6
268	s17	s50	4	1.0
269	s23	s56	4	1.0
270	s58	s58	5	1.0
271	s64	s64	5	1.0

Homework Reinforcement Learning¶

Cooking Chef¶

Introduzione¶

Classi necessarie alla codifica e funzioni ausiliarie¶

Creazione griglia del problema¶

Funzioni ausiliarie¶

Classe MDP¶

Classe Model-free¶

Parte A¶

Creazione insieme di stati $S$¶

a1) Provide a concise description of the states of the MDP¶

a2) How many states are in this MDP? (i.e. what is $|S|$)¶

Creazione insieme di azioni $A$¶

b1) Provide a concise description of the actions of the MDP¶

b2) How many actions are in this MDP? (i.e. what is $|A|$)¶

Creazione insieme di transizioni $P$¶

c) What is the dimensionality of the transition function $P$?¶

d) Report the transition function $P$ for any state $s$ and action $a$ in a tabular format¶

$P[\text{left}, S, S']$¶

$P[\text{right}, S, S']$¶

$P[\text{up}, S, S']$¶

$P[\text{down}, S, S']$¶

$P[\text{take_egg_beater}, S, S']$¶

$P[\text{cook_eggs}, S, S']$¶

Creazione insieme di rewards $R$¶

e) Describe a reward function $R: S \times A \times S$ and a value of $\gamma$ that will lead to an optimal policy¶

Tentativo di risoluzione mediante Policy Iteration¶

f) Does $\gamma \in (0, 1)$ affect the optimal policy in this case? Explain why¶

g) How many possible policies are there? (all policies, not just optimal policies)¶

h) Specifica di uno scenario Model-free¶

Preparazione scenario Model-free¶

Implementazione scenario Model-free¶

Risoluzione mediante Monte Carlo Online Control¶

Stampa della policy trovata¶

i) Is the computed policy deterministic or stochastic?¶

j) Is there any advantage to having a stochastic policy? Explain¶

Parte B¶

Creazione insieme di stati $S$¶

Creazione insieme di azioni $A$¶

Creazione insieme di transizioni $P$ con effetti probabilistici¶

Creazione insieme di rewards $R$¶

Specifica del parametro $\gamma$¶

Tentativo di risoluzione mediante Policy Iteration¶

a) Report the transition function $P$ for any state $s$ and action $a \in A$¶

$P[\text{left}, S, S']$¶

$P[\text{right}, S, S']$¶

$P[\text{up}, S, S']$¶

$P[\text{down}, S, S']$¶

$P[\text{take_egg_beater}, S, S']$¶

$P[\text{cook_eggs}, S, S']$¶

b) Does the optimal policy change compared to Part A? Justify your answer¶

c) Will the value of the optimal policy change? Explain how¶

**g) How many possible policies are there? (all policies, not just optimal policies)**¶

	Sorgente (s)	Destinazione (s')	Azione (a)	Probabilità (P[a, s, s'])
0	s1	s0	0	1.0
1	s2	s1	0	1.0
2	s3	s2	0	1.0
3	s5	s4	0	1.0
4	s6	s5	0	1.0
5	s7	s6	0	1.0
6	s9	s8	0	1.0
7	s10	s9	0	1.0
8	s11	s10	0	1.0
9	s12	s11	0	1.0
10	s13	s12	0	1.0
11	s14	s13	0	1.0
12	s16	s15	0	1.0
13	s19	s18	0	1.0
14	s20	s19	0	1.0
15	s22	s21	0	1.0
16	s24	s23	0	1.0
17	s26	s25	0	1.0
18	s27	s26	0	1.0
19	s28	s27	0	1.0
20	s30	s29	0	1.0
21	s31	s30	0	1.0
22	s32	s31	0	1.0
23	s34	s33	0	1.0
24	s35	s34	0	1.0
25	s36	s35	0	1.0
26	s38	s37	0	1.0
27	s39	s38	0	1.0
28	s40	s39	0	1.0
29	s42	s41	0	1.0
30	s43	s42	0	1.0
31	s44	s43	0	1.0
32	s45	s44	0	1.0
33	s46	s45	0	1.0
34	s47	s46	0	1.0
35	s49	s48	0	1.0
36	s52	s51	0	1.0
37	s53	s52	0	1.0
38	s55	s54	0	1.0
39	s57	s56	0	1.0
40	s59	s58	0	1.0
41	s60	s59	0	1.0
42	s61	s60	0	1.0
43	s63	s62	0	1.0
44	s64	s63	0	1.0
45	s65	s64	0	1.0
46	s0	s1	1	1.0
47	s1	s2	1	1.0
48	s2	s3	1	1.0
49	s4	s5	1	1.0
50	s5	s6	1	1.0
51	s6	s7	1	1.0
52	s8	s9	1	1.0
53	s9	s10	1	1.0
54	s10	s11	1	1.0
55	s11	s12	1	1.0
56	s12	s13	1	1.0
57	s13	s14	1	1.0
58	s15	s16	1	1.0
59	s18	s19	1	1.0
60	s19	s20	1	1.0
61	s21	s22	1	1.0
62	s23	s24	1	1.0
63	s25	s26	1	1.0
64	s26	s27	1	1.0
65	s27	s28	1	1.0
66	s29	s30	1	1.0
67	s30	s31	1	1.0
68	s31	s32	1	1.0
69	s33	s34	1	1.0
70	s34	s35	1	1.0
71	s35	s36	1	1.0
72	s37	s38	1	1.0
73	s38	s39	1	1.0
74	s39	s40	1	1.0
75	s41	s42	1	1.0
76	s42	s43	1	1.0
77	s43	s44	1	1.0
78	s44	s45	1	1.0
79	s45	s46	1	1.0
80	s46	s47	1	1.0
81	s48	s49	1	1.0
82	s51	s52	1	1.0
83	s52	s53	1	1.0
84	s54	s55	1	1.0
85	s56	s57	1	1.0
86	s58	s59	1	1.0
87	s59	s60	1	1.0
88	s60	s61	1	1.0
89	s62	s63	1	1.0
90	s63	s64	1	1.0
91	s64	s65	1	1.0
92	s3	s11	2	1.0
93	s4	s13	2	1.0
94	s5	s14	2	1.0
95	s7	s16	2	1.0
96	s8	s17	2	1.0
97	s11	s20	2	1.0
98	s13	s21	2	1.0
99	s14	s22	2	1.0
100	s15	s23	2	1.0
101	s16	s24	2	1.0
102	s18	s26	2	1.0
103	s19	s27	2	1.0
104	s20	s28	2	1.0
105	s21	s29	2	1.0
106	s22	s30	2	1.0
107	s36	s44	2	1.0
108	s37	s46	2	1.0
109	s38	s47	2	1.0
110	s40	s49	2	1.0
111	s41	s50	2	1.0
112	s44	s53	2	1.0
113	s46	s54	2	1.0
114	s47	s55	2	1.0
115	s48	s56	2	1.0
116	s49	s57	2	1.0
117	s51	s59	2	1.0
118	s52	s60	2	1.0
119	s53	s61	2	1.0
120	s54	s62	2	1.0
121	s55	s63	2	1.0
122	s11	s3	3	1.0
123	s13	s4	3	1.0
124	s14	s5	3	1.0
125	s16	s7	3	1.0
126	s17	s8	3	1.0
127	s20	s11	3	1.0
128	s21	s13	3	1.0
129	s22	s14	3	1.0
130	s23	s15	3	1.0
131	s24	s16	3	1.0
132	s26	s18	3	1.0
133	s27	s19	3	1.0
134	s28	s20	3	1.0
135	s29	s21	3	1.0
136	s30	s22	3	1.0
137	s44	s36	3	1.0
138	s46	s37	3	1.0
139	s47	s38	3	1.0
140	s49	s40	3	1.0
141	s50	s41	3	1.0
142	s53	s44	3	1.0
143	s54	s46	3	1.0
144	s55	s47	3	1.0
145	s56	s48	3	1.0
146	s57	s49	3	1.0
147	s59	s51	3	1.0
148	s60	s52	3	1.0
149	s61	s53	3	1.0
150	s62	s54	3	1.0
151	s63	s55	3	1.0
152	s17	s50	4	1.0
153	s23	s56	4	1.0
154	s58	s58	5	1.0
155	s64	s64	5	1.0

	Sorgente (s)	Destinazione (s')	Azione (a)	Probabilità (P[a, s, s'])
0	s1	s1	0	1.0
1	s2	s2	0	1.0
2	s3	s3	0	0.6
3	s3	s11	0	0.4
4	s5	s5	0	0.6
5	s5	s14	0	0.4
6	s6	s6	0	1.0
7	s7	s7	0	0.6
8	s7	s16	0	0.4
9	s9	s9	0	1.0
10	s10	s10	0	1.0
11	s11	s3	0	0.6
12	s11	s20	0	0.4
13	s12	s12	0	1.0
14	s13	s4	0	0.6
15	s13	s21	0	0.4
16	s14	s5	0	0.6
17	s14	s22	0	0.4
18	s16	s7	0	0.6
19	s16	s24	0	0.4
20	s19	s19	0	0.6
21	s19	s27	0	0.4
22	s20	s11	0	0.6
23	s20	s28	0	0.4
24	s22	s14	0	0.6
25	s22	s30	0	0.4
26	s24	s16	0	0.6
27	s24	s24	0	0.4
28	s26	s18	0	0.6
29	s26	s26	0	0.4
30	s27	s19	0	0.6
31	s27	s27	0	0.4
32	s28	s20	0	0.6
33	s28	s28	0	0.4
34	s30	s22	0	0.6
35	s30	s30	0	0.4
36	s31	s31	0	1.0
37	s32	s32	0	1.0
38	s34	s34	0	1.0
39	s35	s35	0	1.0
40	s36	s36	0	0.6
41	s36	s44	0	0.4
42	s38	s38	0	0.6
43	s38	s47	0	0.4
44	s39	s39	0	1.0
45	s40	s40	0	0.6
46	s40	s49	0	0.4
47	s42	s42	0	1.0
48	s43	s43	0	1.0
49	s44	s36	0	0.6
50	s44	s53	0	0.4
51	s45	s45	0	1.0
52	s46	s37	0	0.6
53	s46	s54	0	0.4
54	s47	s38	0	0.6
55	s47	s55	0	0.4
56	s49	s40	0	0.6
57	s49	s57	0	0.4
58	s52	s52	0	0.6
59	s52	s60	0	0.4
60	s53	s44	0	0.6
61	s53	s61	0	0.4
62	s55	s47	0	0.6
63	s55	s63	0	0.4
64	s57	s49	0	0.6
65	s57	s57	0	0.4
66	s59	s51	0	0.6
67	s59	s59	0	0.4
68	s60	s52	0	0.6
69	s60	s60	0	0.4
70	s61	s53	0	0.6
71	s61	s61	0	0.4
72	s63	s55	0	0.6
73	s63	s63	0	0.4
74	s64	s64	0	1.0
75	s65	s65	0	1.0
76	s0	s0	1	1.0
77	s1	s1	1	1.0
78	s2	s2	1	1.0
79	s4	s4	1	0.4
80	s4	s13	1	0.6
81	s5	s5	1	0.4
82	s5	s14	1	0.6
83	s6	s6	1	1.0
84	s8	s8	1	0.4
85	s8	s17	1	0.6
86	s9	s9	1	1.0
87	s10	s10	1	1.0
88	s11	s3	1	0.4
89	s11	s20	1	0.6
90	s12	s12	1	1.0
91	s13	s4	1	0.4
92	s13	s21	1	0.6
93	s15	s15	1	0.4
94	s15	s23	1	0.6
95	s18	s18	1	0.4
96	s18	s26	1	0.6
97	s19	s19	1	0.4
98	s19	s27	1	0.6
99	s21	s13	1	0.4
100	s21	s29	1	0.6
101	s23	s15	1	0.4
102	s23	s23	1	0.6
103	s25	s25	1	1.0
104	s26	s18	1	0.4
105	s26	s26	1	0.6
106	s27	s19	1	0.4
107	s27	s27	1	0.6
108	s29	s21	1	0.4
109	s29	s29	1	0.6
110	s30	s22	1	0.4
111	s30	s30	1	0.6
112	s31	s31	1	1.0
113	s33	s33	1	1.0
114	s34	s34	1	1.0
115	s35	s35	1	1.0
116	s37	s37	1	0.4
117	s37	s46	1	0.6
118	s38	s38	1	0.4
119	s38	s47	1	0.6
120	s39	s39	1	1.0
121	s41	s41	1	0.4
122	s41	s50	1	0.6
123	s42	s42	1	1.0
124	s43	s43	1	1.0
125	s44	s36	1	0.4
126	s44	s53	1	0.6
127	s45	s45	1	1.0
128	s46	s37	1	0.4
129	s46	s54	1	0.6
130	s48	s48	1	0.4
131	s48	s56	1	0.6
132	s51	s51	1	0.4
133	s51	s59	1	0.6
134	s52	s52	1	0.4
135	s52	s60	1	0.6
136	s54	s46	1	0.4
137	s54	s62	1	0.6
138	s56	s48	1	0.4
139	s56	s56	1	0.6
140	s58	s58	1	1.0
141	s59	s51	1	0.4
142	s59	s59	1	0.6
143	s60	s52	1	0.4
144	s60	s60	1	0.6
145	s62	s54	1	0.4
146	s62	s62	1	0.6
147	s63	s55	1	0.4
148	s63	s63	1	0.6
149	s64	s64	1	1.0
150	s3	s2	2	0.6
151	s3	s3	2	0.4
152	s4	s4	2	0.6
153	s4	s5	2	0.4
154	s5	s4	2	0.6
155	s5	s6	2	0.4
156	s7	s6	2	0.6
157	s7	s7	2	0.4
158	s8	s8	2	0.6
159	s8	s9	2	0.4
160	s11	s10	2	0.6
161	s11	s12	2	0.4
162	s13	s12	2	0.6
163	s13	s14	2	0.4
164	s14	s13	2	0.6
165	s14	s14	2	0.4
166	s15	s15	2	0.6
167	s15	s16	2	0.4
168	s16	s15	2	0.6
169	s16	s16	2	0.4
170	s18	s18	2	0.6
171	s18	s19	2	0.4
172	s19	s18	2	0.6
173	s19	s20	2	0.4
174	s20	s19	2	0.6
175	s20	s20	2	0.4
176	s21	s21	2	0.6
177	s21	s22	2	0.4
178	s22	s21	2	0.6
179	s22	s22	2	0.4
180	s36	s35	2	0.6
181	s36	s36	2	0.4
182	s37	s37	2	0.6
183	s37	s38	2	0.4
184	s38	s37	2	0.6
185	s38	s39	2	0.4
186	s40	s39	2	0.6
187	s40	s40	2	0.4
188	s41	s41	2	0.6
189	s41	s42	2	0.4
190	s44	s43	2	0.6
191	s44	s45	2	0.4
192	s46	s45	2	0.6
193	s46	s47	2	0.4
194	s47	s46	2	0.6
195	s47	s47	2	0.4
196	s48	s48	2	0.6
197	s48	s49	2	0.4
198	s49	s48	2	0.6
199	s49	s49	2	0.4
200	s51	s51	2	0.6
201	s51	s52	2	0.4
202	s52	s51	2	0.6
203	s52	s53	2	0.4
204	s53	s52	2	0.6
205	s53	s53	2	0.4
206	s54	s54	2	0.6
207	s54	s55	2	0.4
208	s55	s54	2	0.6
209	s55	s55	2	0.4
210	s11	s10	3	0.4
211	s11	s12	3	0.6
212	s13	s12	3	0.4
213	s13	s14	3	0.6
214	s14	s13	3	0.4
215	s14	s14	3	0.6
216	s16	s15	3	0.4
217	s16	s16	3	0.6
218	s17	s17	3	1.0
219	s20	s19	3	0.4
220	s20	s20	3	0.6
221	s21	s21	3	0.4
222	s21	s22	3	0.6
223	s22	s21	3	0.4
224	s22	s22	3	0.6
225	s23	s23	3	0.4
226	s23	s24	3	0.6
227	s24	s23	3	0.4
228	s24	s24	3	0.6
229	s26	s25	3	0.4
230	s26	s27	3	0.6
231	s27	s26	3	0.4
232	s27	s28	3	0.6
233	s28	s27	3	0.4
234	s28	s28	3	0.6
235	s29	s29	3	0.4
236	s29	s30	3	0.6
237	s30	s29	3	0.4
238	s30	s31	3	0.6
239	s44	s43	3	0.4
240	s44	s45	3	0.6
241	s46	s45	3	0.4
242	s46	s47	3	0.6
243	s47	s46	3	0.4
244	s47	s47	3	0.6
245	s49	s48	3	0.4
246	s49	s49	3	0.6
247	s50	s50	3	1.0
248	s53	s52	3	0.4
249	s53	s53	3	0.6
250	s54	s54	3	0.4
251	s54	s55	3	0.6
252	s55	s54	3	0.4
253	s55	s55	3	0.6
254	s56	s56	3	0.4
255	s56	s57	3	0.6
256	s57	s56	3	0.4
257	s57	s57	3	0.6
258	s59	s58	3	0.4
259	s59	s60	3	0.6
260	s60	s59	3	0.4
261	s60	s61	3	0.6
262	s61	s60	3	0.4
263	s61	s61	3	0.6
264	s62	s62	3	0.4
265	s62	s63	3	0.6
266	s63	s62	3	0.4
267	s63	s64	3	0.6
268	s17	s50	4	1.0
269	s23	s56	4	1.0
270	s58	s58	5	1.0
271	s64	s64	5	1.0